from pathlib import Path
import pandas as pd

DATA = Path('/srv/data/uk-butterflies')

species = pd.read_csv(DATA / 'bfly_species_list.csv')             # 22 species + names
indices = pd.read_csv(DATA / 'bfly_garden_collated_indices.csv')  # yearly index
trends  = pd.read_csv(DATA / 'bfly_garden_trends.csv')            # overall % trend

# Attach the readable common name to every table via the 5-letter species code
names = species.set_index('species')['spname']
indices['spname'] = indices['species'].map(names)
trends['spname']  = trends['species'].map(names)

print('species :', species.shape)
print('indices :', indices.shape, '| years', indices['year'].min(), '-', indices['year'].max(),
      '|', indices['species'].nunique(), 'species')
print('trends  :', trends.shape)
species

species : (22, 4)
indices : (308, 12) | years 2007 - 2020 | 22 species
trends  : (66, 9)

# Confirm the anchor: every species' lci equals 2.0 in 2007
assert (indices.loc[indices['year'] == 2007, 'lci'].round(6) == 2.0).all()

# Abundance relative to the 2007 baseline (1.0 = unchanged since 2007)
indices['rel2007'] = 10 ** (indices['lci'] - 2)

# Monitoring effort has grown a lot over the years -- worth knowing
effort = indices.groupby('year')[['nsite']].first()
print('Gardens monitored: {} in 2007  ->  {} in 2020'.format(
    int(effort.loc[2007, 'nsite']), int(effort.loc[2020, 'nsite'])))

indices[['species', 'spname', 'year', 'col_index', 'lci', 'rel2007']].head()

Gardens monitored: 749 in 2007  ->  5113 in 2020

import matplotlib.pyplot as plt
import numpy as np

# Whole-series garden trend, one row per species
gbw = (trends[(trends['scheme'] == 'GBW') & (trends['years'] == '2007-2020')]
       .sort_values('trend'))

colors = np.where(gbw['trend'] >= 0, '#2a9d8f', '#e76f51')  # teal up, red down
fig, ax = plt.subplots(figsize=(9, 7))
ax.barh(gbw['spname'], gbw['trend'], color=colors)

# Mark statistically significant trends with a star
for y, (t, s) in enumerate(zip(gbw['trend'], gbw['sig'])):
    if pd.notna(s):
        ax.text(t + (6 if t >= 0 else -6), y, '*', va='center',
                ha='left' if t >= 0 else 'right', fontsize=14, color='#444')

ax.axvline(0, color='0.3', lw=0.8)
ax.set_xlabel('Change in garden abundance 2007-2020 (%)')
ax.set_title('UK garden butterflies: winners and losers (2007-2020)\n* = statistically significant')
plt.tight_layout()
plt.show()

print('Only species that DECLINED:', gbw.loc[gbw['trend'] < 0, 'spname'].tolist())

Only species that DECLINED: ['Wall']

# Pick the 3 biggest risers, the single decliner, and a mid-pack species
top3   = gbw.tail(3)['species'].tolist()
bottom = gbw.head(1)['species'].tolist()
picks  = bottom + top3

fig, ax = plt.subplots(figsize=(9, 5.5))
for code in picks:
    d = indices[indices['species'] == code].sort_values('year')
    ax.plot(d['year'], d['rel2007'], marker='o', label=d['spname'].iloc[0])

ax.axhline(1.0, color='0.5', ls='--', lw=1)  # 2007 baseline
ax.set_xlabel('Year')
ax.set_ylabel('Abundance relative to 2007  (1.0 = 2007 level)')
ax.set_title('Trajectories of the biggest risers and the one decliner')
ax.legend(title='Species', fontsize=9)
plt.tight_layout()
plt.show()

wide = indices.pivot(index='year', columns='spname', values='rel2007')
community = wide.mean(axis=1)                 # average across the 22 species
n_down = (wide.diff() < 0).sum(axis=1)        # how many species fell vs last year

fig, (a1, a2) = plt.subplots(2, 1, figsize=(9, 6.5), sharex=True,
                             gridspec_kw={'height_ratios': [2, 1]})
a1.plot(community.index, community.values, marker='o', color='#264653')
a1.axhline(1.0, color='0.5', ls='--', lw=1)
a1.set_ylabel('Community index\n(mean rel. to 2007)')
a1.set_title('A wet, cold 2012 pulled almost every garden species down at once')

worst = int(n_down.idxmax())
bars = a2.bar(n_down.index, n_down.values,
              color=['#e76f51' if y == worst else '#adb5bd' for y in n_down.index])
a2.set_ylabel('# species\nfalling')
a2.set_xlabel('Year')
a2.set_ylim(0, 22)
for a in (a1, a2):
    a.axvline(worst, color='#e76f51', lw=1, ls=':')
plt.tight_layout()
plt.show()

print('Worst shared year: {} -- {} of 22 species declined, community index = {:.2f}'
      .format(worst, int(n_down.max()), community.loc[worst]))

Worst shared year: 2012 -- 18 of 22 species declined, community index = 0.93

import seaborn as sns

# Order species by their overall trend so risers and fallers group together
order = gbw['spname'].tolist()
heat = np.log10(wide[order]).T          # log10(rel-to-2007): 0 = baseline

fig, ax = plt.subplots(figsize=(10, 7))
sns.heatmap(heat, cmap='RdYlGn', center=0, ax=ax,
            cbar_kws={'label': 'log10(abundance relative to 2007)'})
ax.axvline(list(wide.index).index(2012) + 0.5, color='black', lw=1.5)
ax.set_xlabel('Year')
ax.set_ylabel('Species (ordered by overall trend)')
ax.set_title('Every species x every year -- red = below 2007, green = above')
plt.tight_layout()
plt.show()

both = (trends[trends['years'] == '2011-2020']
        .pivot(index='spname', columns='scheme', values='trend')
        .dropna())

fig, ax = plt.subplots(figsize=(7, 7))
ax.scatter(both['BMS'], both['GBW'], color='#5b5f97', zorder=3)
lim = [both.min().min() - 10, both.max().max() + 10]
ax.plot(lim, lim, ls='--', color='0.5')      # y = x : agreement line
ax.axhline(0, color='0.8'); ax.axvline(0, color='0.8')
for sp, r in both.iterrows():
    ax.annotate(sp, (r['BMS'], r['GBW']), fontsize=7, alpha=0.8)
ax.set_xlabel('Countryside trend 2011-2020 (UKBMS, %)')
ax.set_ylabel('Garden trend 2011-2020 (GBW, %)')
ax.set_title('Garden vs countryside trends (points above the line = better in gardens)')
plt.tight_layout()
plt.show()

total = 0
for chunk in pd.read_csv(file, chunksize=1_000_000):
    total += len(chunk)

# ⚠️ RESTORE: this DISCARDS YOUR CHANGES to this notebook and resets it to the original.
# 1. Uncomment the line below (remove the #)   2. Run this cell
# 3. Then: menu File → Reload Notebook from Disk

# !git -C ~/citizen-science-data fetch -q origin && git -C ~/citizen-science-data checkout origin/main -- uk-butterflies.ipynb && echo "Restored. Now: File → Reload Notebook from Disk"

file	what it holds
`bfly_species_list.csv`	the 22 species: 5-letter code, common name, scientific name
`bfly_garden_collated_indices.csv`	a yearly abundance index per species, 2007-2020
`bfly_garden_trends.csv`	one overall % trend per species, for gardens (GBW) and the countryside (UKBMS)

	taxorder	species	spname	sciname
0	342350	XSMSK	Small Skipper	Thymelicus sylvestris
1	342380	XLASK	Large Skipper	Ochlodes sylvanus
2	342460	XBRIM	Brimstone	Gonepteryx rhamni
3	342470	XLAWH	Large White	Pieris brassicae
4	342480	XSMWH	Small White	Pieris rapae
5	342490	XGVWH	Green-veined White	Pieris napi
6	342500	XORTI	Orange-tip	Anthocharis cardamines
7	342570	XSMCO	Small Copper	Lycaena phlaeas
8	342650	XCOBL	Common Blue	Polyommatus icarus
9	342660	XHOBL	Holly Blue	Celastrina argiolus
10	342720	XREAD	Red Admiral	Vanessa atalanta
11	342730	XPALA	Painted Lady	Vanessa cardui
12	342740	XSMTO	Small Tortoiseshell	Aglais urticae
13	342760	XPEAC	Peacock	Aglais io
14	342770	XCOMM	Comma	Polygonia c-album
15	342860	XSPWO	Speckled Wood	Pararge aegeria
16	342870	XWLBR	Wall	Lasiommata megera
17	342890	XMAWH	Marbled White	Melanargia galathea
18	342910	XGATE	Gatekeeper	Pyronia tithonus
19	342920	XMEBR	Meadow Brown	Maniola jurtina
20	342930	XSMHE	Small Heath	Coenonympha pamphilus
21	342950	XRING	Ringlet	Aphantopus hyperantus

	species	spname	year	col_index	lci	rel2007
0	XBRIM	Brimstone	2007	1.437071	2.000000	1.000000
1	XBRIM	Brimstone	2008	1.068299	1.871215	0.743386
2	XBRIM	Brimstone	2009	1.736926	2.082303	1.208657
3	XBRIM	Brimstone	2010	1.580160	2.041223	1.099570
4	XBRIM	Brimstone	2011	1.552585	2.033577	1.080381

Trends in Butterfly Populations in UK Gardens¶

What's in the dataset¶

Understanding the index columns¶

1. Winners and losers: ranking the overall trend¶

2. Was there a year when everything crashed together?¶

3. Bonus: are gardens telling the same story as the wider countryside?¶

Working with data larger than memory¶

Your turn¶