Observational Data Staging Workflow

# Cell 1: Configuration
from pathlib import Path
from datetime import date

DATASETS = {
    "ctd_mooring_qc": {
        "source_id": "seanoe_113246",
        "doi": "10.17882/113246",
        "dataset_page": "https://www.seanoe.org/data/01021/113246/",
        "url": "https://www.seanoe.org/data/01021/113246/data/127701.zip",
        "file_type": "zip",
    },
    "adcp_mooring_qc": {
        "source_id": "seanoe_113246",
        "doi": "10.17882/113246",
        "dataset_page": "https://www.seanoe.org/data/01021/113246/",
        "url": "https://www.seanoe.org/data/01021/113246/data/127702.zip",
        "file_type": "zip",
    },
    "ctd_profiles_qc": {
        "source_id": "seanoe_110439",
        "doi": "10.17882/110439",
        "dataset_page": "https://www.seanoe.org/data/00993/110439/",
        "url": "https://www.seanoe.org/data/00993/110439/data/124334.xlsx",
        "file_type": "xlsx",
    },
    "discrete_samples_qc": {
        "source_id": "seanoe_110401",
        "doi": "10.17882/110401",
        "dataset_page": "https://www.seanoe.org/data/00992/110401/",
        "url": "https://www.seanoe.org/data/00992/110401/data/124314.xlsx",
        "file_type": "xlsx",
    },
}

RUN_DATE = date.today().isoformat()

PROJECT_DATA_ROOT = Path("/anvil/projects/x-ees250129/x-uheede/C-Star-in-Hvalfjordur/data")
RAW_RUN_DIR = PROJECT_DATA_ROOT / "raw" / "seanoe_hvalfjordur" / RUN_DATE
STAGED_RUN_DIR = PROJECT_DATA_ROOT / "staged" / "seanoe_hvalfjordur" / RUN_DATE
MANIFEST_CSV = PROJECT_DATA_ROOT / "staged" / "seanoe_hvalfjordur" / f"manifest_{RUN_DATE}.csv"

RAW_RUN_DIR.mkdir(parents=True, exist_ok=True)
STAGED_RUN_DIR.mkdir(parents=True, exist_ok=True)

print("RAW_RUN_DIR:", RAW_RUN_DIR)
print("STAGED_RUN_DIR:", STAGED_RUN_DIR)
print("Datasets configured:", list(DATASETS))

# Cell 2: Helpers
import csv
import hashlib
import shutil
import urllib.request
import zipfile

def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str:
    h = hashlib.sha256()
    with path.open("rb") as f:
        while True:
            chunk = f.read(chunk_size)
            if not chunk:
                break
            h.update(chunk)
    return h.hexdigest()

def download_file(url: str, dest: Path) -> Path:
    if dest.exists():
        print(f"Already exists, skipping download: {dest.name}")
        return dest

    dest.parent.mkdir(parents=True, exist_ok=True)
    print(f"Downloading: {url}")
    with urllib.request.urlopen(url) as response, dest.open("wb") as out:
        shutil.copyfileobj(response, out)
    print(f"Saved: {dest}")
    return dest

def unzip_file(zip_path: Path, extract_to: Path) -> list[Path]:
    extract_to.mkdir(parents=True, exist_ok=True)
    with zipfile.ZipFile(zip_path, "r") as zf:
        zf.extractall(extract_to)
        extracted = [extract_to / name for name in zf.namelist()]
    return extracted

def copy_file_to_stage(source_path: Path, staged_dir: Path) -> list[Path]:
    staged_dir.mkdir(parents=True, exist_ok=True)
    dest = staged_dir / source_path.name
    shutil.copy2(source_path, dest)
    return [dest]

# Cell 3: Download raw files
downloaded = {}

for key, cfg in DATASETS.items():
    raw_dataset_dir = RAW_RUN_DIR / cfg["source_id"]
    dest = raw_dataset_dir / f"{key}.{cfg['file_type']}"
    downloaded[key] = download_file(cfg["url"], dest)

downloaded

# Cell 4: Stage downloaded files (extract ZIP, copy other formats)
staged_index = {}

for key, local_path in downloaded.items():
    cfg = DATASETS[key]
    out_dir = STAGED_RUN_DIR / cfg["source_id"] / key

    if cfg["file_type"] == "zip":
        staged_files = unzip_file(local_path, out_dir)
        action = "extracted"
    else:
        staged_files = copy_file_to_stage(local_path, out_dir)
        action = "copied"

    staged_index[key] = staged_files
    print(f"{key}: {action} {len(staged_files)} item(s) to {out_dir}")

# Cell 6 (optional): quick preview of what was staged
for source_dir in sorted(STAGED_RUN_DIR.iterdir()):
    if source_dir.is_dir():
        sample = list(source_dir.rglob("*"))[:8]
        print(f"\n{source_dir.name} (showing up to 8 paths):")
        for p in sample:
            print(" -", p.relative_to(STAGED_RUN_DIR))