fix: resolve automated review findings

Signed-off-by: Magnus Hedemark <magnus919@pm.me>

data-cleaning/scripts/profile_dataset.py

@@ -4,6 +4,7 @@ import argparse
 import csv
 import hashlib
 import json
+import math
 import pathlib
 import sys
 from collections import Counter
@@ -69,7 +70,9 @@ def profile(path, delimiter=None, max_rows=None, include_values=False):
         numbers = []
         for value in nonempty:
             try:
-                numbers.append(float(str(value).strip()))
+                parsed = float(str(value).strip())
+                if math.isfinite(parsed):
+                    numbers.append(parsed)
             except (TypeError, ValueError):
                 pass
         field = {
@@ -118,11 +121,15 @@ def main():
     except (OSError, UnicodeError, ValueError) as exc:
         print(f"Error: cannot profile input: {exc}", file=sys.stderr)
         return 2
-    text = json.dumps(result, indent=2, sort_keys=True) + "\n"
-    if args.output == "-":
-        print(text, end="")
-    else:
-        pathlib.Path(args.output).write_text(text, encoding="utf-8")
+    try:
+        text = json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n"
+        if args.output == "-":
+            print(text, end="")
+        else:
+            pathlib.Path(args.output).write_text(text, encoding="utf-8")
+    except (OSError, IsADirectoryError, TypeError, ValueError) as exc:
+        print(f"Error: cannot write profile: {exc}", file=sys.stderr)
+        return 2
     return 0
 
 

data-cleaning/scripts/reconcile_dataset.py

@@ -1,27 +1,87 @@
 #!/usr/bin/env python3
-"""Compare two CSV/TSV datasets by shape, key uniqueness, and aggregate columns."""
-import argparse, csv, hashlib, json, pathlib, sys
+"""Compare two CSV/TSV datasets by shape, key uniqueness, and aggregates."""
+import argparse
+import csv
+import hashlib
+import json
+import math
+import pathlib
+import sys
 from collections import Counter
 
+
 def read(path, delimiter=None):
-    raw=pathlib.Path(path).read_bytes(); text=raw.decode("utf-8-sig")
-    if delimiter is None: delimiter=csv.Sniffer().sniff(text[:8192],delimiters=",\t;|").delimiter
-    rows=list(csv.DictReader(text.splitlines(),delimiter=delimiter)); return rows, hashlib.sha256(raw).hexdigest()
+    source = pathlib.Path(path)
+    raw = source.read_bytes()
+    with source.open("r", encoding="utf-8-sig", newline="") as handle:
+        sample = handle.read(8192)
+        handle.seek(0)
+        try:
+            selected = delimiter or csv.Sniffer().sniff(sample, delimiters=",\t;|").delimiter
+        except csv.Error:
+            selected = delimiter or ","
+        rows = list(csv.DictReader(handle, delimiter=selected))
+    return rows, hashlib.sha256(raw).hexdigest()
+
 
 def main():
-    ap=argparse.ArgumentParser(description="Reconcile two delimited datasets without modifying them.")
-    ap.add_argument("before"); ap.add_argument("after"); ap.add_argument("--key",action="append",required=True); ap.add_argument("--sum",dest="sums",action="append",default=[]); ap.add_argument("--output",default="-"); args=ap.parse_args()
-    try: before,bh=read(args.before); after,ah=read(args.after)
-    except (OSError,UnicodeError,ValueError) as e: print(f"Error: cannot read dataset: {e}",file=sys.stderr); return 2
-    def keys(rows): return [tuple(r.get(k) for k in args.key) for r in rows]
-    bk,ak=keys(before),keys(after); result={"before":{"rows":len(before),"sha256":bh,"duplicate_keys":sum(n-1 for n in Counter(bk).values() if n>1)},"after":{"rows":len(after),"sha256":ah,"duplicate_keys":sum(n-1 for n in Counter(ak).values() if n>1)},"key":args.key,"missing_keys":len(set(bk)-set(ak)),"new_keys":len(set(ak)-set(bk)),"sums":{}}
-    for col in args.sums:
+    parser = argparse.ArgumentParser(description="Reconcile two delimited datasets without modifying them.")
+    parser.add_argument("before")
+    parser.add_argument("after")
+    parser.add_argument("--key", action="append", required=True)
+    parser.add_argument("--sum", dest="sums", action="append", default=[])
+    parser.add_argument("--delimiter")
+    parser.add_argument("--output", default="-")
+    args = parser.parse_args()
+    try:
+        before_path = pathlib.Path(args.before).resolve(strict=True)
+        after_path = pathlib.Path(args.after).resolve(strict=True)
+        before, before_hash = read(before_path, args.delimiter)
+        after, after_hash = read(after_path, args.delimiter)
+        if args.output != "-":
+            output_path = pathlib.Path(args.output).resolve()
+            if output_path.exists() and (output_path.samefile(before_path) or output_path.samefile(after_path)):
+                print("Error: --output must not overwrite or alias an input", file=sys.stderr)
+                return 2
+    except (OSError, UnicodeError, ValueError, csv.Error) as exc:
+        print(f"Error: cannot read dataset: {exc}", file=sys.stderr)
+        return 2
+
+    def keys(rows):
+        return [tuple(row.get(key) for key in args.key) for row in rows]
+
+    before_keys, after_keys = keys(before), keys(after)
+    result = {
+        "before": {"rows": len(before), "sha256": before_hash, "duplicate_keys": sum(n - 1 for n in Counter(before_keys).values() if n > 1)},
+        "after": {"rows": len(after), "sha256": after_hash, "duplicate_keys": sum(n - 1 for n in Counter(after_keys).values() if n > 1)},
+        "key": args.key,
+        "missing_keys": len(set(before_keys) - set(after_keys)),
+        "new_keys": len(set(after_keys) - set(before_keys)),
+        "sums": {},
+    }
+    for column in args.sums:
         def total(rows):
-            return sum(float(r[col]) for r in rows if r.get(col) not in (None,""))
-        try: result["sums"][col]={"before":total(before),"after":total(after),"delta":total(after)-total(before)}
-        except (KeyError,ValueError) as e: print(f"Error: cannot sum {col}: {e}",file=sys.stderr); return 2
-    text=json.dumps(result,indent=2,sort_keys=True)+"\n"
-    if args.output=="-": print(text,end="")
-    else: pathlib.Path(args.output).write_text(text,encoding="utf-8")
+            values = [float(row[column]) for row in rows if row.get(column) not in (None, "")]
+            if any(not math.isfinite(value) for value in values):
+                raise ValueError("non-finite numeric value")
+            return sum(values)
+        try:
+            before_total, after_total = total(before), total(after)
+        except (KeyError, TypeError, ValueError) as exc:
+            print(f"Error: cannot sum {column}: {exc}", file=sys.stderr)
+            return 2
+        result["sums"][column] = {"before": before_total, "after": after_total, "delta": after_total - before_total}
+    try:
+        text = json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n"
+        if args.output == "-":
+            print(text, end="")
+        else:
+            pathlib.Path(args.output).write_text(text, encoding="utf-8")
+    except (OSError, IsADirectoryError, TypeError, ValueError) as exc:
+        print(f"Error: cannot write reconciliation: {exc}", file=sys.stderr)
+        return 2
     return 0
-if __name__=="__main__": raise SystemExit(main())
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

data-cleaning/scripts/test_reconcile_dataset.py

@@ -13,16 +13,35 @@ def test_reconcile_dataset():
         root = Path(directory)
         before = root / "before.csv"
         after = root / "after.csv"
-        before.write_text("id,amount\n001,2\n002,3\n", encoding="utf-8")
-        after.write_text("id,amount\n001,2\n003,4\n", encoding="utf-8")
+        before.write_text('id,amount,note\n001,2,"line one\nline two"\n002,3,ok\n', encoding="utf-8")
+        after.write_text('id,amount,note\n001,2,"line one\nline two"\n003,4,new\n', encoding="utf-8")
         result = json.loads(subprocess.check_output([sys.executable, str(SCRIPT), str(before), str(after), "--key", "id", "--sum", "amount"], text=True))
         assert result["missing_keys"] == 1
         assert result["new_keys"] == 1
         assert result["sums"]["amount"]["delta"] == 1.0
 
 
+def test_reconcile_edge_cases():
+    with tempfile.TemporaryDirectory() as directory:
+        root = Path(directory)
+        before = root / "before.csv"
+        after = root / "after.csv"
+        before.write_text("id\n1\n", encoding="utf-8")
+        after.write_text("id\n1\n", encoding="utf-8")
+        output_alias = root / "alias.csv"
+        output_alias.hardlink_to(before)
+        overwrite = subprocess.run([sys.executable, str(SCRIPT), str(before), str(after), "--key", "id", "--output", str(output_alias)], capture_output=True, text=True)
+        assert overwrite.returncode == 2
+        assert before.read_text(encoding="utf-8") == "id\n1\n"
+        empty = root / "empty.csv"
+        empty.write_text("id\n", encoding="utf-8")
+        result = subprocess.check_output([sys.executable, str(SCRIPT), str(empty), str(empty), "--key", "id"], text=True)
+        assert json.loads(result)["before"]["rows"] == 0
+
+
 def run():
     test_reconcile_dataset()
+    test_reconcile_edge_cases()
 
 
 if __name__ == "__main__":