Moved stuff around, changed the way I'm storing milestones data
This commit is contained in:
@@ -1,118 +1,210 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
import Papa, { ParseRemoteConfig, ParseResult } from 'papaparse';
|
||||
import { Milestone } from '../models/milestone';
|
||||
import { Column } from '../enums/column';
|
||||
import { Subject } from 'rxjs';
|
||||
import { ReplaySubject } from 'rxjs';
|
||||
import { KeyValue } from '../models/key-value';
|
||||
import { Csv } from '../enums/csv';
|
||||
import { Milestone } from '../enums/milestone';
|
||||
|
||||
@Injectable({ providedIn: 'root' })
|
||||
export class DataService {
|
||||
private _data: Milestone[] = [];
|
||||
private _data: Record<string, any>[] = [];
|
||||
private data = new ReplaySubject<Record<string, any>[]>();
|
||||
readonly Data = this.data.asObservable();
|
||||
|
||||
private counties = new Subject<string[]>();
|
||||
private _counties = new Map<string, string>();
|
||||
private counties = new ReplaySubject<KeyValue[]>();
|
||||
readonly Counties = this.counties.asObservable();
|
||||
|
||||
private schools = new Subject<string[]>();
|
||||
private _schools = new Map<string, string>();
|
||||
private schools = new ReplaySubject<KeyValue[]>();
|
||||
readonly Schools = this.schools.asObservable();
|
||||
|
||||
private years = new Subject<number[]>();
|
||||
private _years = Array(2023 - 2015 + 1)
|
||||
.fill(0)
|
||||
.map((_, index) => 2015 + index)
|
||||
.filter((y) => y !== 2020);
|
||||
private years = new ReplaySubject<KeyValue[]>();
|
||||
readonly Years = this.years.asObservable();
|
||||
|
||||
private grades = new Subject<number[]>();
|
||||
private _grades = Array(8 - 3 + 1)
|
||||
.fill(0)
|
||||
.map((_, index) => 3 + index);
|
||||
private grades = new ReplaySubject<KeyValue[]>();
|
||||
readonly Grades = this.grades.asObservable();
|
||||
|
||||
private _cohorts = new Map<string, string>();
|
||||
private cohorts = new ReplaySubject<KeyValue[]>();
|
||||
readonly Cohorts = this.cohorts.asObservable();
|
||||
|
||||
constructor() {
|
||||
this.load();
|
||||
}
|
||||
|
||||
private load(): void {
|
||||
let files: { url: string; year: number; grade: number }[] = [];
|
||||
for (let year = 2015; year < 2024; ++year) {
|
||||
if (year !== 2020) {
|
||||
for (let grade = 3; grade < 9; ++grade) {
|
||||
files.push({
|
||||
url: `assets/data/${year}-${grade}.csv`,
|
||||
year: year,
|
||||
grade: grade,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
let count = this._years.length * this._grades.length;
|
||||
this._years.forEach((year: number) => {
|
||||
this._grades.forEach((grade: number) => {
|
||||
Papa.parse(`assets/data/${year}-${grade}.csv`, {
|
||||
download: true,
|
||||
header: false,
|
||||
complete: (results: ParseResult<Record<string, any>>) => {
|
||||
const headers = this.getHeaders((results?.data ?? []).slice(1, 3));
|
||||
const lookup = (x: string, record: Record<string, any>): any => {
|
||||
const idx = headers.findIndex((h: string) => x === h);
|
||||
return (idx > -1 ? record[idx] : '') as any;
|
||||
};
|
||||
const cohort = year + 12 - grade;
|
||||
this._cohorts.set(`${cohort}`, `Class of ${cohort}`);
|
||||
|
||||
let count = files.length;
|
||||
files.forEach((file: { url: string; year: number; grade: number }) => {
|
||||
Papa.parse(file.url, {
|
||||
download: true,
|
||||
header: false,
|
||||
complete: (results: ParseResult<Record<string, any>>) => {
|
||||
(results?.data ?? []).forEach(
|
||||
(record: Record<string, any>, index: number) => {
|
||||
if (index < 3) return;
|
||||
this._data.push(
|
||||
new Milestone({
|
||||
Year: file.year,
|
||||
Grade: file.grade,
|
||||
County: record[Column.County],
|
||||
School: record[Column.School],
|
||||
ELA: {
|
||||
Count: record[Column.ELA_Count],
|
||||
Mean: record[Column.ELA_Mean],
|
||||
L0: record[Column.ELA_0],
|
||||
L1: record[Column.ELA_1],
|
||||
L2: record[Column.ELA_2],
|
||||
L3: record[Column.ELA_3],
|
||||
},
|
||||
Math: {
|
||||
Count: record[Column.Math_Count],
|
||||
Mean: record[Column.Math_Mean],
|
||||
L0: record[Column.Math_0],
|
||||
L1: record[Column.Math_1],
|
||||
L2: record[Column.Math_2],
|
||||
L3: record[Column.Math_3],
|
||||
},
|
||||
Science: {
|
||||
Count: record[Column.Sci_Count],
|
||||
Mean: record[Column.Sci_Mean],
|
||||
L0: record[Column.Sci_0],
|
||||
L1: record[Column.Sci_1],
|
||||
L2: record[Column.Sci_2],
|
||||
L3: record[Column.Sci_3],
|
||||
},
|
||||
Social: {
|
||||
Count: record[Column.Soc_Count],
|
||||
Mean: record[Column.Soc_Mean],
|
||||
L0: record[Column.Soc_0],
|
||||
L1: record[Column.Soc_1],
|
||||
L2: record[Column.Soc_2],
|
||||
L3: record[Column.Soc_3],
|
||||
},
|
||||
})
|
||||
const data: Record<string, any>[] = [];
|
||||
|
||||
(results?.data ?? []).forEach((record: Record<string, any>) => {
|
||||
let code = parseInt(lookup(Csv.County_Code, record));
|
||||
// skip non-data rows on the csv
|
||||
if (isNaN(code)) return;
|
||||
|
||||
const ms = {} as Record<string, any>;
|
||||
// const ms = new Milestone({});
|
||||
ms[Milestone.Year] = year;
|
||||
ms[Milestone.Grade] = grade;
|
||||
ms[Milestone.Cohort] = cohort;
|
||||
ms[Milestone.County] = lookup(Csv.County_Code, record);
|
||||
ms[Milestone.School] = lookup(Csv.School_Code, record);
|
||||
ms[Milestone.ELACount] = lookup(Csv.ELA_Count, record);
|
||||
ms[Milestone.ELAMean] = lookup(Csv.ELA_Mean, record);
|
||||
ms[Milestone.ELA0] = lookup(Csv.ELA_L0, record);
|
||||
ms[Milestone.ELA1] = lookup(Csv.ELA_L1, record);
|
||||
ms[Milestone.ELA2] = lookup(Csv.ELA_L2, record);
|
||||
ms[Milestone.ELA3] = lookup(Csv.ELA_L3, record);
|
||||
ms[Milestone.MathCount] = lookup(Csv.Math_Count, record);
|
||||
ms[Milestone.MathMean] = lookup(Csv.Math_Mean, record);
|
||||
ms[Milestone.Math0] = lookup(Csv.Math_L0, record);
|
||||
ms[Milestone.Math1] = lookup(Csv.Math_L1, record);
|
||||
ms[Milestone.Math2] = lookup(Csv.Math_L2, record);
|
||||
ms[Milestone.Math3] = lookup(Csv.Math_L3, record);
|
||||
ms[Milestone.SciCount] = lookup(Csv.Science_Count, record);
|
||||
ms[Milestone.SciMean] = lookup(Csv.Science_Mean, record);
|
||||
ms[Milestone.Sci0] = lookup(Csv.Science_L0, record);
|
||||
ms[Milestone.Sci1] = lookup(Csv.Science_L1, record);
|
||||
ms[Milestone.Sci2] = lookup(Csv.Science_L2, record);
|
||||
ms[Milestone.Sci3] = lookup(Csv.Science_L3, record);
|
||||
ms[Milestone.SocCount] = lookup(Csv.Social_Count, record);
|
||||
ms[Milestone.SocMean] = lookup(Csv.Social_Mean, record);
|
||||
ms[Milestone.Soc0] = lookup(Csv.Social_L0, record);
|
||||
ms[Milestone.Soc1] = lookup(Csv.Social_L1, record);
|
||||
ms[Milestone.Soc2] = lookup(Csv.Social_L2, record);
|
||||
ms[Milestone.Soc3] = lookup(Csv.Social_L3, record);
|
||||
data.push(ms);
|
||||
|
||||
this._counties.set(
|
||||
lookup(Csv.County_Code, record),
|
||||
lookup(Csv.County_Label, record)
|
||||
);
|
||||
}
|
||||
);
|
||||
--count;
|
||||
if (count === 0) this.loadComplete();
|
||||
},
|
||||
error: (error: any) => {
|
||||
console.error(error);
|
||||
--count;
|
||||
if (count === 0) this.loadComplete();
|
||||
},
|
||||
} as ParseRemoteConfig);
|
||||
this._schools.set(
|
||||
`${lookup(Csv.County_Code, record)}-${lookup(
|
||||
Csv.School_Code,
|
||||
record
|
||||
)}`,
|
||||
lookup(Csv.School_Label, record)
|
||||
);
|
||||
});
|
||||
|
||||
[
|
||||
{ sort: Milestone.ELAMean, rank: Milestone.ELARank },
|
||||
{ sort: Milestone.MathMean, rank: Milestone.MathRank },
|
||||
{ sort: Milestone.SciMean, rank: Milestone.SciRank },
|
||||
{ sort: Milestone.SocMean, rank: Milestone.SocRank },
|
||||
].forEach((m) => {
|
||||
data
|
||||
.sort(
|
||||
(a: Record<string, any>, b: Record<string, any>) =>
|
||||
+a[m.sort] - b[m.sort]
|
||||
)
|
||||
.forEach(
|
||||
(a: Record<string, any>, index: number) => (a[m.rank] = index)
|
||||
);
|
||||
});
|
||||
|
||||
this._data.push(...data);
|
||||
|
||||
--count;
|
||||
if (count === 0) this.loadComplete();
|
||||
},
|
||||
error: (error: any) => {
|
||||
console.error(error);
|
||||
--count;
|
||||
if (count === 0) this.loadComplete();
|
||||
},
|
||||
} as ParseRemoteConfig);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private getHeaders(data: Record<string, any>[]): string[] {
|
||||
const headers: string[] = [];
|
||||
data.forEach((record: Record<string, any>) => {
|
||||
let prv = '';
|
||||
Object.keys(record).forEach((key: string, index: number) => {
|
||||
let value = (record[key] ?? '')
|
||||
.replace(/\n/g, '')
|
||||
.replace(/- EOG/g, '');
|
||||
if (value.length === 0) value = prv;
|
||||
|
||||
headers[index] = `${headers[index] ?? ''} ${value}`.trim();
|
||||
prv = value;
|
||||
});
|
||||
});
|
||||
return headers;
|
||||
}
|
||||
|
||||
private loadComplete(): void {
|
||||
this.cohorts.next(
|
||||
[...this._cohorts]
|
||||
.map(([key, value]) => new KeyValue({ key: key, value: value }))
|
||||
.sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value))
|
||||
);
|
||||
this.counties.next(
|
||||
[...new Set(this._data.map((data: Milestone) => data.County))].sort()
|
||||
[...this._counties]
|
||||
.map(
|
||||
([code, label]) =>
|
||||
new KeyValue({
|
||||
key: code,
|
||||
value: label
|
||||
.toLowerCase()
|
||||
.replace(/\b[a-z]/g, (c) => c.toUpperCase()),
|
||||
})
|
||||
)
|
||||
.sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value))
|
||||
);
|
||||
this.schools.next(
|
||||
[...new Set(this._data.map((data: Milestone) => data.School))].sort()
|
||||
[...this._schools]
|
||||
.map(
|
||||
([code, label]) =>
|
||||
new KeyValue({
|
||||
key: code,
|
||||
value: label
|
||||
.toLowerCase()
|
||||
.replace(/\b[a-z]/g, (c) => c.toUpperCase()),
|
||||
})
|
||||
)
|
||||
.sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value))
|
||||
);
|
||||
this.years.next(
|
||||
[...new Set(this._data.map((data: Milestone) => data.Year))].sort()
|
||||
this._years
|
||||
.map(
|
||||
(year: number) =>
|
||||
new KeyValue({ key: year.toString(), value: year.toString() })
|
||||
)
|
||||
.sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value))
|
||||
);
|
||||
this.grades.next(
|
||||
[...new Set(this._data.map((data: Milestone) => data.Grade))].sort()
|
||||
this._grades
|
||||
.map(
|
||||
(grade: number) =>
|
||||
new KeyValue({ key: grade.toString(), value: grade.toString() })
|
||||
)
|
||||
.sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value))
|
||||
);
|
||||
this.data.next(this._data);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user