fix: move null is zero to data source, fix bugs

migrations/1772117945260_data_source_null_is_zero.ts

@@ -0,0 +1,13 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable("dataSource")
+    .addColumn("nullIsZero", "boolean", (col) => col.defaultTo(false))
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema.alterTable("dataSource").dropColumn("nullIsZero").execute();
+}

src/app/(private)/data-sources/[id]/components/ConfigurationForm.tsx

@@ -55,6 +55,7 @@ export default function ConfigurationForm({
   const [isPublic, setIsPublic] = useState(dataSource.public);
 
   const [naIsNull, setNaIsNull] = useState(dataSource.naIsNull || false);
+  const [nullIsZero, setNullIsZero] = useState(dataSource.nullIsZero || false);
 
   // Form state
   const [loading, setLoading] = useState(false);
@@ -69,12 +70,26 @@ export default function ConfigurationForm({
         // Notify parent of autoImport status change
         onAutoImportChange?.(autoImport);
 
-        // Invalidate webhook status to refetch and show/hide errors
-        await queryClient.invalidateQueries({
-          queryKey: trpc.dataSource.checkWebhookStatus.queryKey({
-            dataSourceId: dataSource.id,
+        // Invalidate cached data source queries and derived stats so
+        // map rendering and other parts of the app reflect the new config
+        await Promise.all([
+          queryClient.invalidateQueries({
+            queryKey: trpc.dataSource.checkWebhookStatus.queryKey({
+              dataSourceId: dataSource.id,
+            }),
           }),
-        });
+          queryClient.invalidateQueries({
+            queryKey: trpc.dataSource.byId.queryKey({
+              dataSourceId: dataSource.id,
+            }),
+          }),
+          queryClient.invalidateQueries({
+            queryKey: trpc.dataSource.listReadable.queryKey(),
+          }),
+          queryClient.invalidateQueries({
+            queryKey: trpc.area.stats.queryKey(),
+          }),
+        ]);
 
         if (redirectToParent) {
           router.push(`/data-sources/${dataSource.id}`);
@@ -111,6 +126,7 @@ export default function ConfigurationForm({
       dateFormat,
       public: isPublic,
       naIsNull,
+      nullIsZero,
     });
   };
 
@@ -193,6 +209,17 @@ export default function ConfigurationForm({
         <Switch checked={naIsNull} onCheckedChange={(v) => setNaIsNull(v)} />
       </FormFieldWrapper>
 
+      <FormFieldWrapper
+        label="Treat empty values as zero"
+        hint="Switch this on if numeric columns in your data source are blank when the value is zero."
+        isHorizontal
+      >
+        <Switch
+          checked={nullIsZero}
+          onCheckedChange={(v) => setNullIsZero(v)}
+        />
+      </FormFieldWrapper>
+
       <FormFieldWrapper
         label="Share this data with the Movement Data Library"
         hint="Warning: this will make the data publicly available."

src/app/map/[id]/components/Legend.tsx

@@ -145,6 +145,11 @@ export default function Legend() {
           .filter((v) => v && v !== "null" && v !== "undefined"),
       );
 
+      const valueLabels =
+        dataSource?.columnMetadata.find(
+          (c) => c.name === viewConfig.areaDataColumn,
+        )?.valueLabels || {};
+
       return (
         <div className="flex flex-col gap-1.5 w-full py-1">
           {Object.keys(colorScheme.colorMap)
@@ -155,15 +160,25 @@ export default function Legend() {
               }
               return a < b ? -1 : 1;
             })
-            .map((key) => (
-              <div className="flex items-center gap-2 text-xs" key={key}>
-                <div
-                  className="w-3 h-3 flex-shrink-0 border border-neutral-300"
-                  style={{ backgroundColor: colorScheme.colorMap[key] }}
-                />
-                <span>{key}</span>
-              </div>
-            ))}
+            .map((key) => {
+              let label = valueLabels[key];
+              if (
+                !label &&
+                areaStats?.primary?.columnType === ColumnType.Number &&
+                Number(key) === 0
+              ) {
+                label = valueLabels[""];
+              }
+              return (
+                <div className="flex items-center gap-2 text-xs" key={key}>
+                  <div
+                    className="w-3 h-3 flex-shrink-0 border border-neutral-300"
+                    style={{ backgroundColor: colorScheme.colorMap[key] }}
+                  />
+                  <span>{label || key}</span>
+                </div>
+              );
+            })}
           <div className="flex items-center gap-2 text-xs">
             <div
               className="w-3 h-3 flex-shrink-0 border border-neutral-300"
@@ -293,9 +308,20 @@ export default function Legend() {
     const hasValueLabels = Object.keys(valueLabels).length > 0;
 
     if (hasValueLabels) {
-      numTicks = Object.keys(valueLabels).length;
-      denom = Math.max(numTicks - 1, 1);
-      values = Object.keys(valueLabels).map(Number).toSorted();
+      const numericKeys = Object.keys(valueLabels)
+        .map((key) => Number(key))
+        .filter(
+          (v) =>
+            Number.isFinite(v) &&
+            v >= colorScheme.minValue &&
+            v <= colorScheme.maxValue,
+        )
+        .toSorted((a, b) => a - b);
+      if (numericKeys.length) {
+        values = numericKeys;
+        numTicks = values.length;
+        denom = Math.max(numTicks - 1, 1);
+      }
     }
 
     return (

src/app/map/[id]/components/controls/VisualisationPanel/VisualisationPanel.tsx

@@ -168,8 +168,6 @@ export default function VisualisationPanel({
       dataSource?.geocodingConfig,
       viewConfig.areaSetGroupCode,
     );
-  const showEmptyZeroSwitch = !isCount && columnOneIsNumber;
-
   const showStyle = !viewConfig.areaDataSecondaryColumn;
   const canSelectColorScale = isCount || columnOneIsNumber;
   const canSelectColorScheme = canSelectColorScale && !isCategorical;
@@ -443,25 +441,6 @@ export default function VisualisationPanel({
               </Select>
             </>
           )}
-
-          {showEmptyZeroSwitch && (
-            <div className="col-span-2 flex items-center gap-2">
-              <Label
-                htmlFor="choropleth-empty-zero-switch"
-                className="text-sm text-muted-foreground font-normal"
-              >
-                Treat empty values as zero
-              </Label>
-
-              <Switch
-                id="choropleth-empty-zero-switch"
-                checked={Boolean(viewConfig.areaDataNullIsZero)}
-                onCheckedChange={(v) =>
-                  updateViewConfig({ areaDataNullIsZero: v })
-                }
-              />
-            </div>
-          )}
         </div>
         {!viewConfig.areaDataSourceId && (
           <div className="flex items-center gap-2">
@@ -532,7 +511,7 @@ export default function VisualisationPanel({
                   }
                 >
                   <SelectTrigger
-                    className="w-full min-w-0"
+                    className={cn("w-full min-w-0", SELECT_TO_BUTTON_CLASSES)}
                     id="color-scale-type-select"
                   >
                     <SelectValue placeholder="Choose color scale...">

src/app/map/[id]/data.ts

@@ -50,7 +50,6 @@ export const useAreaStats = () => {
     areaDataColumn: column,
     areaDataSecondaryColumn: secondaryColumn,
     areaDataSourceId: dataSourceId,
-    areaDataNullIsZero: nullIsZero,
     areaSetGroupCode,
   } = viewConfig;
 
@@ -118,7 +117,6 @@ export const useAreaStats = () => {
         dataSourceId,
         column: columnOrCount,
         secondaryColumn: secondaryColumn,
-        nullIsZero,
         includeColumns,
         boundingBox: requiresBoundingBox ? boundingBox : null,
       },
@@ -134,7 +132,6 @@ export const useAreaStats = () => {
     dataSourceId,
     column,
     secondaryColumn,
-    nullIsZero,
     includeColumns,
   ]);
 

src/app/map/[id]/utils/mapView.ts

@@ -9,7 +9,6 @@ export const createNewViewConfig = (): MapViewConfig => {
   return {
     areaDataSourceId: "",
     areaDataColumn: "",
-    areaDataNullIsZero: true,
     areaSetGroupCode: undefined,
     mapStyleName: MapStyleName.Light,
     showLabels: true,

src/server/models/DataSource.ts

@@ -255,6 +255,7 @@ export const dataSourceSchema = z.object({
   createdAt: z.date(),
   dateFormat: z.string(),
   naIsNull: z.boolean().optional(),
+  nullIsZero: z.boolean().optional(),
 });
 
 export type DataSource = z.infer<typeof dataSourceSchema>;

src/server/models/MapView.ts

@@ -130,7 +130,6 @@ export const mapViewConfigSchema = z.object({
   areaDataSourceId: z.string(),
   areaDataColumn: z.string(),
   areaDataSecondaryColumn: z.string().optional(),
-  areaDataNullIsZero: z.boolean().optional(),
   areaSetGroupCode: areaSetGroupCode.nullish(),
   secondaryAreaSetCode: areaSetCode.nullish(),
   choroplethOpacityPct: z.number().optional(),

src/server/stats/index.ts

@@ -45,7 +45,6 @@ export const getAreaStats = async ({
   calculationType,
   column,
   secondaryColumn,
-  nullIsZero,
   includeColumns,
   boundingBox = null,
 }: {
@@ -54,7 +53,6 @@ export const getAreaStats = async ({
   calculationType: CalculationType;
   column: string;
   secondaryColumn?: string;
-  nullIsZero?: boolean;
   includeColumns?: string[] | null;
   boundingBox?: BoundingBox | null;
 }): Promise<AreaStats> => {
@@ -110,7 +108,6 @@ export const getAreaStats = async ({
       areaSetCode,
       calculationType,
       column,
-      nullIsZero,
       boundingBox,
     });
     const valueRange = getValueRange(primaryStats.stats);
@@ -128,7 +125,6 @@ export const getAreaStats = async ({
       areaSetCode,
       calculationType,
       column: secondaryColumn,
-      nullIsZero,
       boundingBox,
     });
     const secondaryValueRange = getValueRange(secondaryStats.stats);
@@ -225,6 +221,7 @@ export const getMaxColumnByArea = async (
         ${caseWhen.end()} AS max_column
       FROM data_record
       WHERE data_source_id = ${dataSourceId}
+        AND geocode_result->'areas'->>${areaSetCode} IS NOT NULL
         AND ${getBoundingBoxSQL(boundingBox)}
         AND ${hasNonNullValueCondition}
         AND ${caseWhen.end()} IS NOT NULL
@@ -254,32 +251,36 @@ const getColumnValueByArea = async ({
   areaSetCode,
   calculationType,
   column,
-  nullIsZero,
   boundingBox,
 }: {
   dataSource: DataSource;
   areaSetCode: AreaSetCode;
   calculationType: CalculationType;
   column: string;
-  nullIsZero: boolean | undefined;
   boundingBox: BoundingBox | null;
 }) => {
   const columnDef = dataSource.columnDefs.find((c) => c.name === column);
   if (!columnDef) {
     throw new Error(`Data source column not found: ${column}`);
   }
 
-  // Coalesce empty values to 0 if set
-  const numberSelect = nullIsZero
+  // Coalesce numeric empty values to 0 if set
+  const numberSelect = dataSource.nullIsZero
     ? sql`(COALESCE(NULLIF(json->>${column}, ''), '0'))::float`
     : sql`(NULLIF(json->>${column}, ''))::float`;
 
+  // Coalesce mode empty values to 0 if the column is numeric and nullIsZero is set
+  const modeSelect =
+    columnDef.type === ColumnType.Number && dataSource.nullIsZero
+      ? sql`MODE () WITHIN GROUP (ORDER BY COALESCE(NULLIF(json->>${column}, ''), '0'))`
+      : sql`MODE () WITHIN GROUP (ORDER BY NULLIF(json->>${column}, ''))`;
+
   // Select is always MODE for ColumnType !== Number
   const isMode =
     calculationType === CalculationType.Mode ||
     columnDef.type !== ColumnType.Number;
   const valueSelect = isMode
-    ? sql`MODE () WITHIN GROUP (ORDER BY json->>${column})`.as("value")
+    ? modeSelect.as("value")
     : db.fn(calculationType, [numberSelect]).as("value");
 
   const query = db
@@ -289,6 +290,7 @@ const getColumnValueByArea = async ({
       valueSelect,
     ])
     .where("dataRecord.dataSourceId", "=", dataSource.id)
+    .where(sql<boolean>`geocode_result->'areas'->>${areaSetCode} IS NOT NULL`)
     .where(getBoundingBoxSQL(boundingBox))
     .groupBy("areaCode");
 
@@ -310,6 +312,7 @@ export const getRecordCountByArea = async (
         ({ fn }) => fn.countAll().as("value"),
       ])
       .where("dataRecord.dataSourceId", "=", dataSourceId)
+      .where(sql<boolean>`geocode_result->'areas'->>${areaSetCode} IS NOT NULL`)
       .where(getBoundingBoxSQL(boundingBox))
       .groupBy("areaCode");
 

src/server/trpc/routers/area.ts

@@ -38,7 +38,6 @@ export const areaRouter = router({
         calculationType: z.nativeEnum(CalculationType),
         column: z.string(),
         secondaryColumn: z.string().optional(),
-        nullIsZero: z.boolean().optional(),
         includeColumns: z.array(z.string()).optional().nullable(),
         boundingBox: boundingBoxSchema.nullish(),
       }),

src/server/trpc/routers/dataSource.ts

@@ -239,6 +239,7 @@ export const dataSourceRouter = router({
         dateFormat: input.dateFormat,
         public: input.public,
         naIsNull: input.naIsNull,
+        nullIsZero: input.nullIsZero,
       } as DataSourceUpdate;
 
       logger.info(

tests/feature/importDataSource.test.ts

@@ -222,8 +222,12 @@ describe("importDataSource tests", () => {
 
     expect(columnDefs).toEqual([
       { name: "Code", type: ColumnType.String },
+      { name: "Con %", type: ColumnType.Number },
       { name: "Electorate", type: ColumnType.Number },
+      { name: "Lab %", type: ColumnType.Number },
       { name: "Name", type: ColumnType.String },
+      { name: "Segment", type: ColumnType.Number },
+      { name: "Segment name", type: ColumnType.String },
     ]);
 
     const areaStats = await getAreaStats({
@@ -247,17 +251,21 @@ describe("importDataSource tests", () => {
     });
 
     expect(stats).toEqual([
-      {
-        areaCode: "E14001070",
-        value: 71002,
-      },
       {
         areaCode: "E14001071",
         value: 79169,
       },
       {
         areaCode: "E14001088",
-        value: 72198,
+        value: 72199,
+      },
+      {
+        areaCode: "E14001092",
+        value: 71002,
+      },
+      {
+        areaCode: "E14001225",
+        value: 72481,
       },
     ]);
   });