from pathlib import Path

DATA = Path('/srv/data/galaxy-zoo-desi')

for f in sorted(DATA.rglob('*')):
    if f.is_file():
        print(f"{f.relative_to(DATA)}  ({f.stat().st_size/1e6:,.1f} MB)")

external_catalog.parquet  (1,616.7 MB)
gz_desi_deep_learning_catalog_advanced.parquet  (7,558.1 MB)
gz_desi_deep_learning_catalog_friendly.csv  (1,612.4 MB)
gz_desi_deep_learning_catalog_friendly.parquet  (658.8 MB)
gz_desi_gzd8_volunteer_core_catalog.parquet  (6.4 MB)
gz_desi_gzd8_volunteer_extended_catalog.parquet  (5.5 MB)

import pyarrow.parquet as pq
from pathlib import Path

DATA = Path("/srv/data/galaxy-zoo-desi")
FRIENDLY = DATA / "gz_desi_deep_learning_catalog_friendly.parquet"
EXTERNAL = DATA / "external_catalog.parquet"

pf = pq.ParquetFile(FRIENDLY)
print(f"{pf.metadata.num_rows:,} galaxies x {pf.metadata.num_columns} columns "
      f"(read from the footer, no data loaded)\n")

# The morphology questions, grouped by their prefix
questions = sorted({c.rsplit("_", 1)[0].split("_")[0]
                    for c in pf.schema.names if c.endswith("_fraction")})
print("Morphology questions with vote fractions:")
for q in questions:
    print("  -", q)

8,689,370 galaxies x 41 columns (read from the footer, no data loaded)

Morphology questions with vote fractions:
  - bar
  - bulge-size
  - disk-edge-on
  - edge-on-bulge
  - has-spiral-arms
  - how-rounded
  - merging
  - smooth-or-featured
  - spiral-arm-count
  - spiral-winding

import duckdb
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_theme(style="whitegrid", context="talk")

con = duckdb.connect()
con.execute("PRAGMA threads=4; PRAGMA memory_limit='2GB'")  # stay well under 4 GB

census = con.execute(f"""
    SELECT CASE
        WHEN "smooth-or-featured_artifact_fraction"
             >= greatest("smooth-or-featured_smooth_fraction",
                         "smooth-or-featured_featured-or-disk_fraction") THEN 'artifact'
        WHEN "smooth-or-featured_smooth_fraction"
             >= "smooth-or-featured_featured-or-disk_fraction"           THEN 'smooth (elliptical)'
        ELSE 'featured / disk'
    END AS morphology,
    count(*) AS n
    FROM read_parquet('{FRIENDLY}')
    GROUP BY 1 ORDER BY n DESC
""").df()

# Distribution of the featured-or-disk vote fraction (0..1), binned on disk
dist = con.execute(f"""
    SELECT round("smooth-or-featured_featured-or-disk_fraction" / 0.05) * 0.05 AS frac,
           count(*) AS n
    FROM read_parquet('{FRIENDLY}')
    GROUP BY 1 ORDER BY 1
""").df()

census["pct"] = 100 * census["n"] / census["n"].sum()
print(census)

            morphology        n        pct
0  smooth (elliptical)  7341109  84.483789
1      featured / disk  1061783  12.219332
2             artifact   286478   3.296879

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))

colors = {"smooth (elliptical)": "#d1495b",
          "featured / disk": "#3b7dd8",
          "artifact": "#8d99ae"}
bars = ax1.barh(census["morphology"], census["n"],
                color=[colors[m] for m in census["morphology"]])
ax1.invert_yaxis()
ax1.set_xlabel("number of galaxies")
ax1.set_title("Dominant morphology (8.7 M galaxies)")
for b, pct in zip(bars, census["pct"]):
    ax1.text(b.get_width(), b.get_y() + b.get_height()/2,
             f"  {pct:.1f}%", va="center", fontsize=13)
ax1.margins(x=0.15)

ax2.bar(dist["frac"], dist["n"], width=0.045, color="#3b7dd8", alpha=0.85)
ax2.set_yscale("log")
ax2.set_xlabel("featured-or-disk vote fraction")
ax2.set_ylabel("galaxies (log scale)")
ax2.set_title("How confident is the model?")

plt.tight_layout()
plt.show()

color_morph = con.execute(f"""
    WITH j AS (
        SELECT e.mag_g_desi - e.mag_r_desi AS g_minus_r,
               f."smooth-or-featured_smooth_fraction"           AS smooth,
               f."smooth-or-featured_featured-or-disk_fraction" AS featured
        FROM read_parquet('{FRIENDLY}') f
        JOIN read_parquet('{EXTERNAL}') e USING (dr8_id)
        WHERE e.mag_g_desi IS NOT NULL AND e.mag_r_desi IS NOT NULL
          AND e.mag_g_desi - e.mag_r_desi BETWEEN 0 AND 2
    )
    SELECT round(g_minus_r / 0.05) * 0.05 AS gr_bin,
           count(*)     AS n,
           avg(smooth)  AS avg_smooth,
           avg(featured) AS avg_featured
    FROM j GROUP BY 1 HAVING count(*) > 500 ORDER BY 1
""").df()

fig, ax = plt.subplots(figsize=(11, 6))
ax.plot(color_morph["gr_bin"], color_morph["avg_smooth"],
        "-o", color="#d1495b", label="smooth (elliptical)")
ax.plot(color_morph["gr_bin"], color_morph["avg_featured"],
        "-o", color="#3b7dd8", label="featured / disk")
ax.set_xlabel("DESI colour  g − r  (redder →)")
ax.set_ylabel("mean vote fraction")
ax.set_title("Redder galaxies are smoother; bluer galaxies are more featured")
ax.legend(title="mean fraction of…")

# faint background: how many galaxies sit in each colour bin
axb = ax.twinx()
axb.bar(color_morph["gr_bin"], color_morph["n"], width=0.045,
        color="grey", alpha=0.12)
axb.set_ylabel("galaxies per bin")
axb.grid(False)

plt.tight_layout()
plt.show()

z_trend = con.execute(f"""
    WITH j AS (
        SELECT e.redshift AS z,
               f."smooth-or-featured_featured-or-disk_fraction" AS featured,
               f."has-spiral-arms_yes_fraction"                 AS spiral
        FROM read_parquet('{FRIENDLY}') f
        JOIN read_parquet('{EXTERNAL}') e USING (dr8_id)
        WHERE e.redshift BETWEEN 0.02 AND 0.35
    )
    SELECT round(z / 0.01) * 0.01 AS z_bin,
           count(*)      AS n,
           avg(featured) AS avg_featured,
           avg(spiral)   AS avg_spiral
    FROM j GROUP BY 1 HAVING count(*) > 500 ORDER BY 1
""").df()

fig, ax = plt.subplots(figsize=(11, 6))
ax.plot(z_trend["z_bin"], z_trend["avg_featured"],
        "-o", color="#3b7dd8", label="featured / disk")
ax.plot(z_trend["z_bin"], z_trend["avg_spiral"],
        "-s", color="#2a9d8f", label="has spiral arms")
ax.set_xlabel("redshift  z  (more distant →)")
ax.set_ylabel("mean vote fraction")
ax.set_title("Detected features fade with distance (an observational bias)")
ax.legend()
plt.tight_layout()
plt.show()

con.close()

total = 0
for chunk in pd.read_csv(file, chunksize=1_000_000):
    total += len(chunk)

# ⚠️ RESTORE: this DISCARDS YOUR CHANGES to this notebook and resets it to the original.
# 1. Uncomment the line below (remove the #)   2. Run this cell
# 3. Then: menu File → Reload Notebook from Disk

# !git -C ~/citizen-science-data fetch -q origin && git -C ~/citizen-science-data checkout origin/main -- galaxy-zoo-desi.ipynb && echo "Restored. Now: File → Reload Notebook from Disk"

File	What it holds
`gz_desi_deep_learning_catalog_friendly.parquet`	The headline product: deep-learning vote fractions for each morphology question (smooth vs featured, spiral arms, bar, bulge, merger…).
`gz_desi_deep_learning_catalog_advanced.parquet`	Same fractions plus 90% confidence intervals and "proportion asked" (7.1 GB).
`external_catalog.parquet`	Cross-matched physical properties: DESI magnitudes, redshift, colour, stellar mass, star-formation rates…
`gz_desi_gzd8_volunteer_*_catalog.parquet`	The raw human volunteer votes for the GZD-8 subset.
`gz_desi_deep_learning_catalog_friendly.csv`	CSV copy of the friendly catalogue.

Galaxy Zoo DESI: Detailed Morphology Classifications¶

What's in the dataset¶

Peek at the schema without loading anything¶

Step 1 — A morphology census with DuckDB¶

Step 2 — The colour–morphology relation¶

Step 3 — A cautionary tale: morphology depends on distance¶

What we found¶

Working with data larger than memory¶

Your turn¶