import { Injectable } from '@angular/core'; import Papa, { ParseRemoteConfig, ParseResult } from 'papaparse'; import { ReplaySubject } from 'rxjs'; import { KeyValue } from '../models/key-value'; import { Csv } from '../enums/csv'; import { Milestone } from '../enums/milestone'; @Injectable({ providedIn: 'root' }) export class DataService { private _data: Record[] = []; private data = new ReplaySubject[]>(1); readonly Data$ = this.data.asObservable(); readonly Data = () => this._data; private _counties = new Map(); private counties = new ReplaySubject(1); readonly Counties$ = this.counties.asObservable(); readonly Counties = () => this._counties; private _schools = new Map(); private schools = new ReplaySubject(1); readonly Schools$ = this.schools.asObservable(); readonly Schools = () => this._schools; private _years = Array(2023 - 2015 + 1) .fill(0) .map((_, index) => 2015 + index); private years = new ReplaySubject(1); readonly Years$ = this.years.asObservable(); readonly Years = () => this._years; private _grades = Array(8 - 3 + 1) .fill(0) .map((_, index) => 3 + index); private grades = new ReplaySubject(1); readonly Grades$ = this.grades.asObservable(); readonly Grades = () => this._grades; private _cohorts = new Map(); private cohorts = new ReplaySubject(1); readonly Cohorts$ = this.cohorts.asObservable(); readonly Cohorts = () => this._cohorts; constructor() { this.load(); } private load(): void { let count = this._years.length * this._grades.length; this._years.forEach((year: number) => { this._grades.forEach((grade: number) => { Papa.parse(`assets/data/${year === 2020 ? 2019 : year}-${grade}.csv`, { download: true, header: false, complete: (results: ParseResult>) => { const h = this.getHeaders((results?.data ?? []).slice(1, 3)); const fs = (x: string, r: Record): string => { const idx = h.findIndex((h: string) => x === h); return (idx > -1 ? r[idx] : '') as string; }; const fn = (x: string, r: Record): number => { const idx = h.findIndex((h: string) => x === h); return +(idx > -1 ? r[idx] : ''); }; const cohort = year + 12 - grade; this._cohorts.set(`${cohort}`, `Class of ${cohort}`); const data: Record[] = []; (results?.data ?? []).forEach((record: Record) => { const code = fn(Csv.County_Code, record); // skip non-data rows on the csv if (isNaN(code)) return; const county = fs(Csv.County_Code, record); const county_label = fs(Csv.County_Label, record); const school_label = fs(Csv.School_Label, record); if (county_label === '' || school_label === '') return; if (!this._counties.has(county)) { this._counties.set(county, county_label); } const school = fs(Csv.School_Code, record); if (!this._schools.has(school)) { this._schools.set(school, school_label); } const ms = {} as Record; ms[Milestone.Year] = year; ms[Milestone.Grade] = grade; ms[Milestone.Cohort] = cohort; ms[Milestone.County] = this._counties.get(county); //fs(Csv.County_Code, record); ms[Milestone.School] = this._schools.get(school); //fs(Csv.School_Code, record); if (year !== 2020) { ms[Milestone.ELACount] = fn(Csv.ELA_Count, record); ms[Milestone.ELAMean] = fn(Csv.ELA_Mean, record); ms[Milestone.ELA0] = fn(Csv.ELA_L0, record); ms[Milestone.ELA1] = fn(Csv.ELA_L1, record); ms[Milestone.ELA2] = fn(Csv.ELA_L2, record); ms[Milestone.ELA3] = fn(Csv.ELA_L3, record); ms[Milestone.MathCount] = fn(Csv.Math_Count, record); ms[Milestone.MathMean] = fn(Csv.Math_Mean, record); ms[Milestone.Math0] = fn(Csv.Math_L0, record); ms[Milestone.Math1] = fn(Csv.Math_L1, record); ms[Milestone.Math2] = fn(Csv.Math_L2, record); ms[Milestone.Math3] = fn(Csv.Math_L3, record); ms[Milestone.SciCount] = fn(Csv.Science_Count, record); ms[Milestone.SciMean] = fn(Csv.Science_Mean, record); ms[Milestone.Sci0] = fn(Csv.Science_L0, record); ms[Milestone.Sci1] = fn(Csv.Science_L1, record); ms[Milestone.Sci2] = fn(Csv.Science_L2, record); ms[Milestone.Sci3] = fn(Csv.Science_L3, record); ms[Milestone.SocCount] = fn(Csv.Social_Count, record); ms[Milestone.SocMean] = fn(Csv.Social_Mean, record); ms[Milestone.Soc0] = fn(Csv.Social_L0, record); ms[Milestone.Soc1] = fn(Csv.Social_L1, record); ms[Milestone.Soc2] = fn(Csv.Social_L2, record); ms[Milestone.Soc3] = fn(Csv.Social_L3, record); } data.push(ms); // this._counties.set( // fs(Csv.County_Code, record), // fs(Csv.County_Label, record) // ); // this._schools.set( // fs(Csv.School_Code, record), // fs(Csv.School_Label, record) // ); }); if (year !== 2020) { [ { sort: Milestone.ELAMean, rank: Milestone.ELARank }, { sort: Milestone.MathMean, rank: Milestone.MathRank }, { sort: Milestone.SciMean, rank: Milestone.SciRank }, { sort: Milestone.SocMean, rank: Milestone.SocRank }, ].forEach((m) => { data .sort( (a: Record, b: Record) => +a[m.sort] - b[m.sort] ) .forEach( (a: Record, index: number) => (a[m.rank] = index) ); }); } this._data.push(...data); --count; if (count === 0) this.loadComplete(); }, error: (error: any) => { console.error(error); --count; if (count === 0) this.loadComplete(); }, } as ParseRemoteConfig); }); }); } private getHeaders(data: Record[]): string[] { const headers: string[] = []; data.forEach((record: Record) => { let prv = ''; Object.keys(record).forEach((key: string, index: number) => { let value = (record[key] ?? '') .replace(/\n/g, '') .replace(/- EOG/g, ''); if (value.length === 0) value = prv; headers[index] = `${headers[index] ?? ''} ${value}`.trim(); prv = value; }); }); return headers; } private loadComplete(): void { this.cohorts.next( [...this._cohorts] .map(([key, value]) => new KeyValue({ key: key, value: value })) .sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value)) ); this.counties.next( [...this._counties] .map( ([code, label]) => new KeyValue({ key: code, value: label .toLowerCase() .replace(/\b[a-z]/g, (c) => c.toUpperCase()), }) ) .sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value)) ); this.schools.next( [...this._schools] .map( ([code, label]) => new KeyValue({ key: code, value: label .toLowerCase() .replace(/\b[a-z]/g, (c) => c.toUpperCase()), }) ) .sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value)) ); this.years.next( this._years .map( (year: number) => new KeyValue({ key: year.toString(), value: year.toString() }) ) .sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value)) ); this.grades.next( this._grades .map( (grade: number) => new KeyValue({ key: grade.toString(), value: grade.toString() }) ) .sort((a: KeyValue, b: KeyValue) => a.value.localeCompare(b.value)) ); this.data.next(this._data); } }