# CODE FOR DOWNLOADING AND CREATING BATCHES FOR THE SYNTHETIC DATA OLD PUNJAB DATASET¶
In [ ]:
import ee
from google.colab import drive
# 1. SETUP
drive.mount('/content/drive', force_remount=True)
try:
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
except:
ee.Authenticate()
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
# --- CONFIGURATION ---
WHEAT_MASK_ASSET = '[REDACTED_FOR_SECURITY]'
START_DATE = '2023-10-01'
END_DATE = '2024-04-30'
OUTPUT_FOLDER = 'PhD/Obj2'
OUTPUT_FILENAME = 'Master_Raw_10k_Stratified'
# --- 2. SAMPLING (Stratified = 5k Wheat / 5k Non-Wheat) ---
print("Generating 10,000 stratified points (5k Wheat, 5k Non-Wheat)...")
mask_img = ee.Image(WHEAT_MASK_ASSET).rename('class')
#region = Punjab Geometry
# This single function replaces: bounds definition, randomPoints, reduceRegion, filter, and merge.
# It guarantees exactly 5000 Wheat (1) and 5000 Non-Wheat (0).
points = mask_img.stratifiedSample(
numPoints=5000, # 5000 per class = 10,000 Total
classBand='class', # Tells GEE to look for 0s and 1s
region=mask_img.geometry(), # Uses the mask bounds automatically
scale=10,
seed=42,
geometries=True # Essential to keep Lat/Lon
)
print(f"Points generated. Starting Processing...")
# --- 3. SATELLITE PROCESSING (The Correct Math from 100-Sample Code) ---
def add_ndvi(image):
return image.addBands(image.normalizedDifference(['B8', 'B4']).rename('NDVI'))
def maskS2clouds(image):
qa = image.select('QA60')
mask = qa.bitwiseAnd(1<<10).eq(0).And(qa.bitwiseAnd(1<<11).eq(0))
return image.updateMask(mask).select(['NDVI']).copyProperties(image, ["system:time_start"])
# *** THE REFINED LEE FILTER ***
# This matches the math in your "Code 1" (100 samples)
def apply_refined_lee(image):
def lee_single(b):
img = image.select(b)
# 1. Calculate Mean & Variance in 3x3 window
mean = img.reduceNeighborhood(ee.Reducer.mean(), ee.Kernel.square(3))
variance = img.reduceNeighborhood(ee.Reducer.variance(), ee.Kernel.square(3))
# 2. Estimate overall variance (Heuristic constant)
overall_var = ee.Image.constant(0.004)
# 3. Calculate Weight (k)
weight = variance.divide(variance.add(overall_var))
# 4. Apply Filter: Mean + k * (Img - Mean)
return mean.add(weight.multiply(img.subtract(mean))).rename(b)
return image.addBands(lee_single('VV'), overwrite=True) \
.addBands(lee_single('VH'), overwrite=True)
# Collections
s2_col = ee.ImageCollection('COPERNICUS/S2_SR_HARMONIZED') \
.filterDate(START_DATE, END_DATE) \
.filterBounds(mask_img.geometry()) \
.filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE', 20)) \
.map(add_ndvi) \
.map(maskS2clouds) \
.select('NDVI')
# S1 (Corrected to include DESCENDING filter)
s1_col = ee.ImageCollection('COPERNICUS/S1_GRD') \
.filterDate(START_DATE, END_DATE) \
.filterBounds(mask_img.geometry()) \
.filter(ee.Filter.eq('instrumentMode', 'IW')) \
.filter(ee.Filter.eq('orbitProperties_pass', 'DESCENDING')) \
.map(apply_refined_lee) \
.select(['VV', 'VH'])
# --- 4. VECTORIZED STACKING (The Fast Way) ---
def create_stack(collection, band_name):
# Get distinct dates
dates = collection.aggregate_array('system:time_start') \
.map(lambda t: ee.Date(t).format('dd-MM-YYYY')).distinct().sort()
date_list = dates.getInfo()
stack_bands = []
for d_str in date_list:
d_ee = ee.Date.parse('dd-MM-YYYY', d_str)
# Mosaic: Flattens overlaps for that day
daily_img = collection.filterDate(d_ee, d_ee.advance(1, 'day')).mosaic()
# Rename to match your Python parser (e.g. NDVI_05-10-2023)
renamed_band = daily_img.rename(f"{band_name}_{d_str}").set('system:time_start', d_ee.millis())
stack_bands.append(renamed_band)
return stack_bands
print("Stacking Images (Creating the 'Layer Cake')...")
ndvi_stack = create_stack(s2_col, 'NDVI')
vv_stack = create_stack(s1_col, 'VV')
vh_stack = create_stack(s1_col, 'VH')
# Combine into one giant image
full_stack_img = ee.Image.cat(ndvi_stack + vv_stack + vh_stack)
# *** CRITICAL: Handling Missing Data ***
# In 100-sample code (loops), missing values became 0 by default.
# In 10k code (vectors), we must explicitely set them to 0.
# This ensures your Python script (which treats 0 as NaN) works perfectly.
full_stack_safe = full_stack_img.unmask(0)
# --- 5. EXTRACTION ---
print("Extracting 10k points (Vectorized)...")
# Add Lat/Lon as explicit properties for the CSV
points_export = points.map(lambda f: f.set({
'lat': f.geometry().coordinates().get(1),
'lon': f.geometry().coordinates().get(0)
}))
# sampleRegions: The efficient extractor
export_fc = full_stack_safe.sampleRegions(
collection=points_export,
scale=10,
geometries=False # Lat/Lon are already properties
)
# --- 6. EXPORT ---
task = ee.batch.Export.table.toDrive(
collection=export_fc,
description=OUTPUT_FILENAME,
folder=OUTPUT_FOLDER,
fileNamePrefix=OUTPUT_FILENAME,
fileFormat='CSV'
)
task.start()
print(f" Export task started! Look for '{OUTPUT_FILENAME}.csv' in Drive in ~30 mins.")
In [ ]:
import ee
from google.colab import drive
# 1. SETUP
drive.mount('/content/drive', force_remount=True)
try:
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
except:
ee.Authenticate()
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
# --- CONFIGURATION ---
WHEAT_MASK_ASSET = '[REDACTED_FOR_SECURITY]'
START_DATE = '2023-10-01'
END_DATE = '2024-04-30'
OUTPUT_FOLDER = 'PhD/Obj1'
OUTPUT_FILENAME = 'Master_Raw_10k_Stratified'
# --- 2. SAMPLING (Stratified) ---
print("Generating 10,000 stratified points (5k Wheat, 5k Non-Wheat)...")
mask_img = ee.Image(WHEAT_MASK_ASSET).rename('class')
points = mask_img.stratifiedSample(
numPoints=5000,
classBand='class',
region=mask_img.geometry(),
scale=10,
seed=42,
geometries=True
)
print(f"Points generated. Starting Processing...")
# --- 3. SATELLITE PROCESSING ---
def add_ndvi(image):
return image.addBands(image.normalizedDifference(['B8', 'B4']).rename('NDVI'))
def maskS2clouds(image):
qa = image.select('QA60')
mask = qa.bitwiseAnd(1<<10).eq(0).And(qa.bitwiseAnd(1<<11).eq(0))
return image.updateMask(mask).select(['NDVI']).copyProperties(image, ["system:time_start"])
def apply_refined_lee(image):
def lee_single(b):
img = image.select(b)
mean = img.reduceNeighborhood(ee.Reducer.mean(), ee.Kernel.square(3))
variance = img.reduceNeighborhood(ee.Reducer.variance(), ee.Kernel.square(3))
overall_var = ee.Image.constant(0.004)
weight = variance.divide(variance.add(overall_var))
return mean.add(weight.multiply(img.subtract(mean))).rename(b)
return image.addBands(lee_single('VV'), overwrite=True) \
.addBands(lee_single('VH'), overwrite=True)
# Collections
s2_col = ee.ImageCollection('COPERNICUS/S2_SR_HARMONIZED') \
.filterDate(START_DATE, END_DATE) \
.filterBounds(mask_img.geometry()) \
.filter(ee.Filter.lt('CLOUDY_PIXEL_PERCENTAGE', 20)) \
.map(add_ndvi) \
.map(maskS2clouds) \
.select('NDVI')
# S1 (Descending Only)
s1_col = ee.ImageCollection('COPERNICUS/S1_GRD') \
.filterDate(START_DATE, END_DATE) \
.filterBounds(mask_img.geometry()) \
.filter(ee.Filter.eq('instrumentMode', 'IW')) \
.filter(ee.Filter.eq('orbitProperties_pass', 'DESCENDING')) \
.map(apply_refined_lee) \
.select(['VV', 'VH'])
# --- 4. VECTORIZED STACKING (The Fix is Here) ---
def create_stack(collection, band_name):
dates = collection.aggregate_array('system:time_start') \
.map(lambda t: ee.Date(t).format('dd-MM-YYYY')).distinct().sort()
date_list = dates.getInfo()
stack_bands = []
for d_str in date_list:
d_ee = ee.Date.parse('dd-MM-YYYY', d_str)
daily_img = collection.filterDate(d_ee, d_ee.advance(1, 'day')).mosaic()
# --- THE FIX: Select ONLY the band we want before renaming ---
# Before: daily_img.rename(...) -> Tried to rename VV and VH at once (Error!)
# Now: daily_img.select(band_name).rename(...) -> Renames only VV or only VH
renamed_band = daily_img.select(band_name).rename(f"{band_name}_{d_str}").set('system:time_start', d_ee.millis())
stack_bands.append(renamed_band)
return stack_bands
print("Stacking Images...")
ndvi_stack = create_stack(s2_col, 'NDVI')
vv_stack = create_stack(s1_col, 'VV') # Now correctly selects 'VV'
vh_stack = create_stack(s1_col, 'VH') # Now correctly selects 'VH'
full_stack_img = ee.Image.cat(ndvi_stack + vv_stack + vh_stack)
# *** Unmask with 0 ***
full_stack_safe = full_stack_img.unmask(0)
# --- 5. EXTRACTION ---
print("Extracting 10k points...")
points_export = points.map(lambda f: f.set({
'lat': f.geometry().coordinates().get(1),
'lon': f.geometry().coordinates().get(0)
}))
export_fc = full_stack_safe.sampleRegions(
collection=points_export,
scale=10,
geometries=False
)
# --- 6. EXPORT ---
task = ee.batch.Export.table.toDrive(
collection=export_fc,
description=OUTPUT_FILENAME,
folder=OUTPUT_FOLDER,
fileNamePrefix=OUTPUT_FILENAME,
fileFormat='CSV'
)
task.start()
print(f" Export task started! Look for '{OUTPUT_FILENAME}.csv' in Drive in ~30 mins.")
In [ ]:
import ee
import time
from datetime import datetime
# Initialize (if not already done)
try:
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
except:
ee.Authenticate()
ee.Initialize(project='[REDACTED_FOR_SECURITY]')
TASK_NAME = 'Master_Raw_10k_Stratified'
print(f" Searching for task: {TASK_NAME}...")
while True:
# Get all tasks
tasks = ee.batch.Task.list()
# Find our specific task
current_task = next((t for t in tasks if t.config['description'] == TASK_NAME), None)
if not current_task:
print(f" Error: No task named '{TASK_NAME}' found! Did you run the Export cell?")
break
status = current_task.status()
state = status['state']
# Calculate time running
start_ms = status.get('start_timestamp_ms', 0)
if start_ms > 0:
run_time_min = (time.time() * 1000 - start_ms) / 60000
time_str = f"{run_time_min:.1f} minutes"
else:
time_str = "Waiting to start..."
# Output status
timestamp = datetime.now().strftime("%H:%M:%S")
if state == 'READY':
print(f"[{timestamp}] Status: READY (Waiting in queue...)")
elif state == 'RUNNING':
print(f"[{timestamp}] Status: RUNNING | Time elapsed: {time_str}")
elif state == 'COMPLETED':
print(f"\n SUCCESS! Task Completed in {time_str}.")
print("You can now run Part 2 to verify the data.")
break
elif state in ['FAILED', 'CANCELLED']:
print(f"\n FAILED. Error Message: {status.get('error_message', 'Unknown Error')}")
break
# Wait 30 seconds before checking again
time.sleep(30)
In [ ]:
import pandas as pd
import numpy as np
import os
# --- CONFIGURATION (UPDATED TO MATCH YOUR DRIVE) ---
# Note: Using 'PhD Obj1' based on your previous logs
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Master_Raw_10k_Stratified.csv'
OUTPUT_DIR = '/content/drive/MyDrive/PhD Obj1/Batches/'
# Create output directory if it doesn't exist
os.makedirs(OUTPUT_DIR, exist_ok=True)
print(" Loading Master File...")
# 1. LOAD THE MASTER FILE
if os.path.exists(INPUT_FILE):
print(f" Found Master File! Reading data...")
df = pd.read_csv(INPUT_FILE)
# 2. SHUFFLE (Crucial Step)
# random_state=42 ensures this shuffle is the same every time you run it
df_shuffled = df.sample(frac=1, random_state=42).reset_index(drop=True)
print(f"Total Data Loaded: {len(df_shuffled)} rows")
# 3. SPLIT INTO BATCHES
# --- Batch 1: 100 samples (The Pilot) ---
batch_1 = df_shuffled.iloc[0:100]
batch_1.to_csv(f"{OUTPUT_DIR}Batch_01_100.csv", index=False)
print(f"Created 'Batch_01_100.csv' (Rows 0-100)")
# --- Batch 2: 500 samples (The First AI Loop) ---
# Starts at 100, takes next 500
batch_2 = df_shuffled.iloc[100:600]
batch_2.to_csv(f"{OUTPUT_DIR}Batch_02_500.csv", index=False)
print(f" Created 'Batch_02_500.csv' (Rows 100-600)")
# --- Batch 3: 1000 samples ---
# Starts at 600, takes next 1000
batch_3 = df_shuffled.iloc[600:1600]
batch_3.to_csv(f"{OUTPUT_DIR}Batch_03_1000.csv", index=False)
print(f" Created 'Batch_03_1000.csv' (Rows 600-1600)")
# --- Reserve: The rest of the data ---
reserve = df_shuffled.iloc[1600:]
reserve.to_csv(f"{OUTPUT_DIR}Reserve_Data.csv", index=False)
print(f" Saved remaining {len(reserve)} samples as 'Reserve_Data.csv'.")
print("\n BATCHING COMPLETE! You are ready for Processing Batch 1.")
else:
print(f" Error: Could not find file at {INPUT_FILE}")
print("Check if your folder is named 'PhD Obj1' or 'PhD/Obj1'.")
BELOW IS THE CODE FOR THE CONVERTING THE VV/VH FROM DB TO LINEAR SCALE , AGGREGRATION , SAVGOL, SAVITZKY FILTER ETC¶
In [ ]:
import pandas as pd
import numpy as np
import io
# CONFIGURATION
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_100.csv'
OUTPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_Aggregated.csv'
try:
df = pd.read_csv(INPUT_FILE)
print(f"Successfully loaded '{INPUT_FILE}'.")
print("--- Treating 0 values as 'No Data' (NaN) ONLY for Sensor Columns ---")
# Apply NaN replacement strictly to sensor columns
sensor_cols = [c for c in df.columns if c.startswith(('NDVI', 'VV', 'VH'))]
df[sensor_cols] = df[sensor_cols].replace(0, np.nan)
print(f"Original data has {df.shape[0]} rows and {df.shape[1]} columns.")
# 2. LINEAR CONVERSION
print("\nConverting VV and VH from dB to linear scale...")
# Identify VV and VH columns
vv_cols_to_convert = [col for col in df.columns if col.startswith('VV_')]
vh_cols_to_convert = [col for col in df.columns if col.startswith('VH_')]
if vv_cols_to_convert:
df[vv_cols_to_convert] = 10**(df[vv_cols_to_convert] / 10)
print(f"Converted {len(vv_cols_to_convert)} VV columns to linear scale.")
if vh_cols_to_convert:
df[vh_cols_to_convert] = 10**(df[vh_cols_to_convert] / 10)
print(f"Converted {len(vh_cols_to_convert)} VH columns to linear scale.")
# 3. IDENTIFY METRIC COLUMNS
ndvi_cols_raw = [col for col in df.columns if col.startswith('NDVI_')]
vv_cols_raw = [col for col in df.columns if col.startswith('VV_')]
vh_cols_raw = [col for col in df.columns if col.startswith('VH_')]
# 4. FILTERING (>80% Missing Values)
print("\nStep 1: Filtering samples with more than 80% missing values...")
# Calculate missing percentage for each metric
miss_ndvi = df[ndvi_cols_raw].isnull().sum(axis=1) / len(ndvi_cols_raw) if ndvi_cols_raw else pd.Series(0, index=df.index)
miss_vv = df[vv_cols_raw].isnull().sum(axis=1) / len(vv_cols_raw) if vv_cols_raw else pd.Series(0, index=df.index)
miss_vh = df[vh_cols_raw].isnull().sum(axis=1) / len(vh_cols_raw) if vh_cols_raw else pd.Series(0, index=df.index)
# Keep rows where ALL metrics have <= 80% missing data
filter_mask = (miss_ndvi <= 0.8) & (miss_vv <= 0.8) & (miss_vh <= 0.8)
df_filtered = df[filter_mask].copy()
print(f"Number of samples after filtering: {len(df_filtered)}")
id_vars = ['system:index', 'class']
# Handle GEE export naming (sometimes 'class' is exported as 'first')
if 'class' not in df_filtered.columns and 'first' in df_filtered.columns:
df_filtered.rename(columns={'first': 'class'}, inplace=True)
value_vars = [col for col in df_filtered.columns if col.startswith(('NDVI', 'VV', 'VH'))]
df_long = pd.melt(df_filtered, id_vars=id_vars, value_vars=value_vars, var_name='metric_date', value_name='value')
print("Step 2: Reshaping data from wide to long format...")
# 6. SEPARATE METRIC AND DATE
df_long[['metric', 'date']] = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long.drop('metric_date', axis=1, inplace=True)
print("Step 3: Separating metric type and date...")
df_processed = df_long.pivot_table(index=['system:index', 'class', 'date'], columns='metric', values='value').reset_index()
df_processed.columns.name = None
print("Step 4: Creating distinct columns for NDVI, VV, and VH...")
# 8. PROCESS DATE
df_processed['date'] = pd.to_datetime(df_processed['date'], format='%d-%m-%Y')
print("Step 5: Processing date information...")
# 9. ASSIGN REPRESENTATIVE DATE (10-Day Bins)
def get_representative_date(date_obj):
day = date_obj.day
if day <= 10: return date_obj.replace(day=5)
elif day <= 20: return date_obj.replace(day=15)
else: return date_obj.replace(day=25)
df_processed['representative_date'] = df_processed['date'].apply(get_representative_date)
print("Step 6: Assigning each row to a 10-day period representative date...")
# 10. AGGREGATE (MEDIAN)
# Group by Farm, Class, and Representative Date -> Calculate Median
aggregation_groups = df_processed.groupby(['system:index', 'class', 'representative_date'])
aggregated_df = aggregation_groups[['NDVI', 'VV', 'VH']].median().reset_index()
print("Step 7: Aggregating data and calculating medians...")
df_melted_agg = aggregated_df.melt(
id_vars=['system:index', 'class', 'representative_date'],
value_vars=['NDVI', 'VV', 'VH'],
var_name='metric',
value_name='value'
)
# Create the new column name, e.g., 'NDVI_05-10-2021'
df_melted_agg['new_col_name'] = (
df_melted_agg['metric'] + '_' +
df_melted_agg['representative_date'].dt.strftime('%d-%m-%Y')
)
# Pivot to the final wide format
df_wide = df_melted_agg.pivot_table(
index=['system:index', 'class'],
columns='new_col_name',
values='value'
).reset_index()
df_wide.columns.name = None
print("Step 8: Pivoting data into the final wide format...")
# 12. SORT COLUMNS CHRONOLOGICALLY
meta_cols = ['system:index', 'class']
metric_cols = [c for c in df_wide.columns if c not in meta_cols]
def sort_key(col_name):
parts = col_name.split('_')
metric = parts[0]
date_str = parts[1]
return (pd.to_datetime(date_str, format='%d-%m-%Y'), metric)
metric_cols.sort(key=sort_key)
final_df = df_wide[meta_cols + metric_cols]
# 13. EXPORT
print("\n--- Aggregation Complete ---")
print("Showing the first 5 rows:")
print(final_df.head(5).iloc[:, :6])
final_df.to_csv(OUTPUT_FILE, index=False)
print(f"\nSuccessfully saved to: '{OUTPUT_FILE}'")
except Exception as e:
print(f"Error: {e}")
In [ ]:
import pandas as pd
import numpy as np
from scipy.interpolate import interp1d
from scipy.signal import savgol_filter
from pathlib import Path
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_Aggregated.csv'
OUTPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_ReadyForAI.csv'
SAVGOL_WINDOW = 5 # Must be odd
SAVGOL_POLY = 2 # Must be less than window
def load_and_prep_data(filepath: Path) -> pd.DataFrame:
"""
Loads the wide-format CSV and converts it to a long format.
It also parses dates and metrics.
"""
if not filepath.exists():
print(f"Error: Input file not found at {filepath}")
return pd.DataFrame()
print(f"Loading data from {filepath}...")
df = pd.read_csv(filepath)
df_long = pd.melt(df, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
# 2. Drop rows where value is NaN (these are not actual acquisitions)
df_long = df_long.dropna(subset=['value'])
# 3. Split 'metric_date' into 'metric' and 'date'
try:
split_cols = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long['metric'] = split_cols[0]
df_long['date'] = pd.to_datetime(split_cols[1], format='%d-%m-%Y')
except Exception as e:
print(f"Error parsing column names. Ensure they are in 'METRIC_DD-MM-YYYY' format.")
print(f"Problematic column name might be: {df_long['metric_date'].iloc[0]}")
print(f"Error details: {e}")
return pd.DataFrame()
# 4. Filter for only the desired metrics
df_long = df_long[df_long['metric'].isin(['NDVI', 'VV', 'VH'])]
print("Applying spike filter to NDVI data (diff > 0.4)...")
# Separate NDVI from VV/VH
ndvi_mask = df_long['metric'] == 'NDVI'
df_ndvi = df_long[ndvi_mask].copy()
df_other = df_long[~ndvi_mask]
if not df_ndvi.empty:
# Sort by point and then date to prepare for .diff()
df_ndvi = df_ndvi.sort_values(by=['system:index', 'class', 'date'])
df_ndvi['value_diff'] = df_ndvi.groupby(['system:index', 'class'])['value'].diff()
# Find rows where the absolute difference is > 0.4
spike_mask = df_ndvi['value_diff'].abs() > 0.4
df_ndvi.loc[spike_mask, 'value'] = np.nan
print(f"Set {spike_mask.sum()} NDVI values to NaN due to > 0.4 difference.")
# Recombine the filtered NDVI data with the VV/VH data
df_long = pd.concat([df_ndvi.drop(columns=['value_diff']), df_other])
print("Data loading and preparation complete.")
return df_long.drop(columns=['metric_date'])
def get_common_date_range(df_long: pd.DataFrame) -> (pd.Timestamp, pd.Timestamp):
"""
Finds the common date range (latest start, earliest end)
across all three metrics.
"""
metric_ranges = df_long.groupby('metric')['date'].agg(['min', 'max'])
common_start = metric_ranges['min'].max()
common_end = metric_ranges['max'].min()
print(f"Common date range found: {common_start.date()} to {common_end.date()}")
return common_start, common_end
def process_time_series(df_long: pd.DataFrame, common_start: pd.Timestamp,
common_end: pd.Timestamp,
window: int, poly: int) -> pd.DataFrame:
"""
Interpolates, resamples, and smooths the time series for each point.
"""
print("Starting time-series processing (interpolation, resampling, smoothing)...")
# 1. Define target dates for resampling (1st and 16th of each month)
target_dates_raw = []
# Start from the 1st of the common_start month
current_date = pd.Timestamp(year=common_start.year, month=common_start.month, day=15)
while current_date <= common_end:
# Add the 1st of the month if it's within the common range
if current_date >= common_start:
target_dates_raw.append(current_date)
# Check the 16th of the month
date_16th = pd.Timestamp(year=current_date.year, month=current_date.month, day=15)
# Add the 16th if it's within the common range
if date_16th >= common_start and date_16th <= common_end:
target_dates_raw.append(date_16th)
# Move to the 1st of the next month
current_date = current_date + pd.offsets.MonthBegin(1)
# Ensure dates are sorted and unique
target_dates = pd.DatetimeIndex(sorted(list(set(target_dates_raw))))
print(target_dates)
if len(target_dates) == 0:
print("Warning: No target dates (1st or 16th) fall within the common date range.")
return pd.DataFrame()
print(f"Resampling to {len(target_dates)} target dates (1st & 16th of month)...")
target_days = (target_dates - common_start).days
results = []
grouped_by_point = df_long.groupby(['system:index', 'class'])
# Unpack both keys
for (point_id, class_val), point_data in grouped_by_point:
# Process each metric for the current point
for metric in ['NDVI', 'VV', 'VH']:
metric_data_raw = point_data[point_data['metric'] == metric].sort_values('date')
# Drop NaNs (from spike filter) before interpolation ---
metric_data = metric_data_raw.dropna(subset=['value'])
# We need at least 2 points to interpolate
if len(metric_data) < 2:
# Not enough data, fill with NaNs
smoothed_values = [np.nan] * len(target_dates)
else:
# Convert original dates to relative numerical values (days from start)
current_days = (metric_data['date'] - common_start).dt.days
current_values = metric_data['value'].values
# 2. Create interpolation function
f_interp = interp1d(current_days, current_values, kind='linear', fill_value='extrapolate')
# 3. Resample/Interpolate at target dates
resampled_values = f_interp(target_days)
# 4. Smooth the resampled data
# Ensure window size is not larger than the data itself
safe_window = min(window, len(resampled_values))
# Ensure window is odd
if safe_window % 2 == 0:
safe_window -= 1
# Ensure polyorder is less than the (safe) window
safe_poly = min(poly, safe_window - 1)
if safe_window > safe_poly and safe_poly >= 0:
smoothed_values = savgol_filter(resampled_values, safe_window, safe_poly)
else:
smoothed_values = resampled_values
# Store the results
for i, date in enumerate(target_dates):
results.append({
'system:index': point_id,
'class': class_val, # Keep the class
'date': date,
'metric': metric,
'value': smoothed_values[i]
})
print("Time-series processing complete.")
return pd.DataFrame(results)
def format_for_output(results_df: pd.DataFrame) -> pd.DataFrame:
"""
Converts the long-format results back to a wide format for saving.
"""
print("Formatting data for output...")
# Create the 'METRIC_DD-MM-YYYY' column name
results_df['col_name'] = results_df['metric'] + '_' + results_df['date'].dt.strftime('%d-%m-%Y')
wide_df = results_df.pivot(index=['system:index', 'class'], columns='col_name', values='value')
# Clean up for CSV export
wide_df = wide_df.reset_index()
wide_df.columns.name = None
# Sort the columns in ascending dates of NDVI, VV and VH
ndvi_cols = [col for col in wide_df.columns if col.startswith('NDVI_')]
ndvi_cols = sorted(ndvi_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vv_cols = [col for col in wide_df.columns if col.startswith('VV_')]
vv_cols = sorted(vv_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vh_cols = [col for col in wide_df.columns if col.startswith('VH_')]
vh_cols = sorted(vh_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
sorted_cols = ndvi_cols + vv_cols + vh_cols
# LOGIC CHANGE: Include 'class' in final output columns
wide_df = wide_df[['system:index', 'class'] + sorted_cols]
return wide_df
# Ensure window is odd, default to 5 if not
if SAVGOL_WINDOW % 2 == 0:
print(f"Warning: SAVGOL_WINDOW was even ({SAVGOL_WINDOW}), setting to {SAVGOL_WINDOW + 1}.")
SAVGOL_WINDOW += 1
if SAVGOL_POLY >= SAVGOL_WINDOW:
print(f"Warning: SAVGOL_POLY ({SAVGOL_POLY}) must be less than SAVGOL_WINDOW ({SAVGOL_WINDOW}).")
# Adjust polyorder to be valid
SAVGOL_POLY = max(0, SAVGOL_WINDOW - 2) # e.g., if window is 3, poly becomes 1
print(f"Adjusting SAVGOL_POLY to {SAVGOL_POLY}.")
input_path = Path(INPUT_FILE)
output_path = Path(OUTPUT_FILE)
df_long = load_and_prep_data(input_path)
if df_long.empty:
print("Processing stopped due to errors in loading data.")
common_start, common_end = get_common_date_range(df_long)
print(common_end, common_start)
if pd.isna(common_start) or pd.isna(common_end):
print("Error: Could not determine a valid common date range. Check your input data.")
if common_start > common_end:
print(f"Error: Common start date ({common_start.date()}) is after common end date ({common_end.date()}).")
print("This can happen if the time series for different metrics do not overlap.")
results_df = process_time_series(df_long, common_start, common_end,
window=SAVGOL_WINDOW,
poly=SAVGOL_POLY)
if results_df.empty:
print("No results were generated. Check intermediate steps.")
output_df = format_for_output(results_df)
output_df.to_csv(output_path, index=False)
print(f"\nSuccessfully processed data and saved to: {output_path}")
# --- PLOTTING SECTION (Visual Verification) ---
import matplotlib.pyplot as plt
from IPython.display import display
# Load DataFrames for Display
df_initial = pd.read_csv(INPUT_FILE)
df_processed = pd.read_csv(OUTPUT_FILE)
print("Initial Data Head:")
display(df_initial.head())
print("Processed Data Head:")
display(df_processed.head())
# Prepare Long Formats for Plotting
# LOGIC CHANGE: Melt with class
df_initial_long = pd.melt(df_initial, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
df_initial_long = df_initial_long.dropna(subset=['value'])
split_cols_initial = df_initial_long['metric_date'].str.split('_', n=1, expand=True)
df_initial_long['metric'] = split_cols_initial[0]
df_initial_long['date'] = pd.to_datetime(split_cols_initial[1], format='%d-%m-%Y')
df_initial_long = df_initial_long[df_initial_long['metric'].isin(['NDVI', 'VV', 'VH'])].drop(columns=['metric_date'])
# LOGIC CHANGE: Melt with class
df_processed_long = pd.melt(df_processed, id_vars=['system:index', 'class'], var_name='metric_date', value_name='value')
split_cols_processed = df_processed_long['metric_date'].str.split('_', n=1, expand=True)
df_processed_long['metric'] = split_cols_processed[0]
df_processed_long['date'] = pd.to_datetime(split_cols_processed[1], format='%d-%m-%Y')
df_processed_long = df_processed_long[df_processed_long['metric'].isin(['NDVI', 'VV', 'VH'])].drop(columns=['metric_date'])
# Select a representative point to plot (e.g., the first point in the list)
plot_index = 0
if not df_initial_long.empty:
point_id = df_initial_long['system:index'].unique()[plot_index]
# Get class for title
class_val = df_initial_long[df_initial_long['system:index'] == point_id]['class'].iloc[0]
# Filter for the selected point
df_initial_point = df_initial_long[df_initial_long['system:index'] == point_id]
df_processed_point = df_processed_long[df_processed_long['system:index'] == point_id]
# Plot metrics
metrics = ['NDVI', 'VV', 'VH']
for metric in metrics:
plt.figure(figsize=(12, 6))
df_init = df_initial_point[df_initial_point['metric'] == metric].sort_values('date')
df_proc = df_processed_point[df_processed_point['metric'] == metric].sort_values('date')
plt.plot(df_init['date'], df_init['value'], label='Initial (Raw)', marker='o', linestyle='--', alpha=0.6)
plt.plot(df_proc['date'], df_proc['value'], label='Processed (Savgol)', marker='x', linestyle='-', linewidth=2)
plt.xlabel('Date')
plt.ylabel('Value')
plt.title(f'{metric} Time Series for Point: {point_id} (Class: {class_val})')
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()
else:
print("No data available to plot.")
BELOW IS THE SAME CODE AS ABOVE THE ONLY CHANGE IS ADDING THE SUB-CLASS AND ALSO THIS CODE IS FOR LABELED DATASET WHICH WE MANUALLY VERIFIED AND LABELED IT¶
In [ ]:
#This code is for labeled data aggregration it includes the subclass
import pandas as pd
import numpy as np
import io
# CONFIGURATION
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_Cleaned_Labeled.csv'
OUTPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_Cleaned_Labeled_Aggregated.csv'
try:
df = pd.read_csv(INPUT_FILE)
print(f"Successfully loaded '{INPUT_FILE}'.")
print("--- Treating 0 values as 'No Data' (NaN) ONLY for Sensor Columns ---")
# Apply NaN replacement strictly to sensor columns
sensor_cols = [c for c in df.columns if c.startswith(('NDVI', 'VV', 'VH'))]
df[sensor_cols] = df[sensor_cols].replace(0, np.nan)
print(f"Original data has {df.shape[0]} rows and {df.shape[1]} columns.")
# 2. LINEAR CONVERSION
print("\nConverting VV and VH from dB to linear scale...")
# Identify VV and VH columns
vv_cols_to_convert = [col for col in df.columns if col.startswith('VV_')]
vh_cols_to_convert = [col for col in df.columns if col.startswith('VH_')]
if vv_cols_to_convert:
df[vv_cols_to_convert] = 10**(df[vv_cols_to_convert] / 10)
print(f"Converted {len(vv_cols_to_convert)} VV columns to linear scale.")
if vh_cols_to_convert:
df[vh_cols_to_convert] = 10**(df[vh_cols_to_convert] / 10)
print(f"Converted {len(vh_cols_to_convert)} VH columns to linear scale.")
# 3. IDENTIFY METRIC COLUMNS
ndvi_cols_raw = [col for col in df.columns if col.startswith('NDVI_')]
vv_cols_raw = [col for col in df.columns if col.startswith('VV_')]
vh_cols_raw = [col for col in df.columns if col.startswith('VH_')]
# 4. FILTERING (>80% Missing Values)
print("\nStep 1: Filtering samples with more than 80% missing values...")
# Calculate missing percentage for each metric
miss_ndvi = df[ndvi_cols_raw].isnull().sum(axis=1) / len(ndvi_cols_raw) if ndvi_cols_raw else pd.Series(0, index=df.index)
miss_vv = df[vv_cols_raw].isnull().sum(axis=1) / len(vv_cols_raw) if vv_cols_raw else pd.Series(0, index=df.index)
miss_vh = df[vh_cols_raw].isnull().sum(axis=1) / len(vh_cols_raw) if vh_cols_raw else pd.Series(0, index=df.index)
# Keep rows where ALL metrics have <= 80% missing data
filter_mask = (miss_ndvi <= 0.8) & (miss_vv <= 0.8) & (miss_vh <= 0.8)
df_filtered = df[filter_mask].copy()
print(f"Number of samples after filtering: {len(df_filtered)}")
id_vars = ['system:index', 'class', 'sub_class']
# Handle GEE export naming (sometimes 'class' is exported as 'first')
if 'class' not in df_filtered.columns and 'first' in df_filtered.columns:
df_filtered.rename(columns={'first': 'class'}, inplace=True)
value_vars = [col for col in df_filtered.columns if col.startswith(('NDVI', 'VV', 'VH'))]
df_long = pd.melt(df_filtered, id_vars=id_vars, value_vars=value_vars, var_name='metric_date', value_name='value')
print("Step 2: Reshaping data from wide to long format...")
# 6. SEPARATE METRIC AND DATE
df_long[['metric', 'date']] = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long.drop('metric_date', axis=1, inplace=True)
print("Step 3: Separating metric type and date...")
df_processed = df_long.pivot_table(index=['system:index', 'class', 'sub_class', 'date'], columns='metric', values='value').reset_index()
df_processed.columns.name = None
print("Step 4: Creating distinct columns for NDVI, VV, and VH...")
# 8. PROCESS DATE
df_processed['date'] = pd.to_datetime(df_processed['date'], format='%d-%m-%Y')
print("Step 5: Processing date information...")
# 9. ASSIGN REPRESENTATIVE DATE (10-Day Bins)
def get_representative_date(date_obj):
day = date_obj.day
if day <= 10: return date_obj.replace(day=5)
elif day <= 20: return date_obj.replace(day=15)
else: return date_obj.replace(day=25)
df_processed['representative_date'] = df_processed['date'].apply(get_representative_date)
print("Step 6: Assigning each row to a 10-day period representative date...")
# 10. AGGREGATE (MEDIAN)
# Group by Farm, Class, and Representative Date -> Calculate Median
aggregation_groups = df_processed.groupby(['system:index', 'class', 'sub_class', 'representative_date'])
aggregated_df = aggregation_groups[['NDVI', 'VV', 'VH']].median().reset_index()
print("Step 7: Aggregating data and calculating medians...")
df_melted_agg = aggregated_df.melt(
id_vars=['system:index', 'class', 'sub_class', 'representative_date'],
value_vars=['NDVI', 'VV', 'VH'],
var_name='metric',
value_name='value'
)
# Create the new column name, e.g., 'NDVI_05-10-2021'
df_melted_agg['new_col_name'] = (
df_melted_agg['metric'] + '_' +
df_melted_agg['representative_date'].dt.strftime('%d-%m-%Y')
)
# Pivot to the final wide format
df_wide = df_melted_agg.pivot_table(index=['system:index', 'class', 'sub_class'], columns='new_col_name', values='value').reset_index()
df_wide.columns.name = None
print("Step 8: Pivoting data into the final wide format...")
# 12. SORT COLUMNS CHRONOLOGICALLY
meta_cols = ['system:index', 'class', 'sub_class']
metric_cols = [c for c in df_wide.columns if c not in meta_cols]
def sort_key(col_name):
parts = col_name.split('_')
metric = parts[0]
date_str = parts[1]
return (pd.to_datetime(date_str, format='%d-%m-%Y'), metric)
metric_cols.sort(key=sort_key)
final_df = df_wide[meta_cols + metric_cols]
# 13. EXPORT
print("\n--- Aggregation Complete ---")
print("Showing the first 5 rows:")
print(final_df.head(5).iloc[:, :6])
final_df.to_csv(OUTPUT_FILE, index=False)
print(f"\nSuccessfully saved to: '{OUTPUT_FILE}'")
except Exception as e:
print(f"Error: {e}")
In [ ]:
#this code is for labeled subclass data of 100 sample
import pandas as pd
import numpy as np
from scipy.interpolate import interp1d
from scipy.signal import savgol_filter
from pathlib import Path
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_Cleaned_Labeled_Aggregated.csv'
OUTPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_01_Cleaned_Labeled_ReadyForAI.csv'
SAVGOL_WINDOW = 5 # Must be odd
SAVGOL_POLY = 2 # Must be less than window
def load_and_prep_data(filepath: Path) -> pd.DataFrame:
"""
Loads the wide-format CSV and converts it to a long format.
It also parses dates and metrics.
"""
if not filepath.exists():
print(f"Error: Input file not found at {filepath}")
return pd.DataFrame()
print(f"Loading data from {filepath}...")
df = pd.read_csv(filepath)
df_long = pd.melt(df, id_vars=['system:index', 'class', 'sub_class'], var_name='metric_date', value_name='value')
# 2. Drop rows where value is NaN (these are not actual acquisitions)
df_long = df_long.dropna(subset=['value'])
# 3. Split 'metric_date' into 'metric' and 'date'
try:
split_cols = df_long['metric_date'].str.split('_', n=1, expand=True)
df_long['metric'] = split_cols[0]
df_long['date'] = pd.to_datetime(split_cols[1], format='%d-%m-%Y')
except Exception as e:
print(f"Error parsing column names. Ensure they are in 'METRIC_DD-MM-YYYY' format.")
print(f"Problematic column name might be: {df_long['metric_date'].iloc[0]}")
print(f"Error details: {e}")
return pd.DataFrame()
# 4. Filter for only the desired metrics
df_long = df_long[df_long['metric'].isin(['NDVI', 'VV', 'VH'])]
print("Applying spike filter to NDVI data (diff > 0.4)...")
# Separate NDVI from VV/VH
ndvi_mask = df_long['metric'] == 'NDVI'
df_ndvi = df_long[ndvi_mask].copy()
df_other = df_long[~ndvi_mask]
if not df_ndvi.empty:
# Sort by point and then date to prepare for .diff()
df_ndvi = df_ndvi.sort_values(by=['system:index', 'class', 'sub_class', 'date'])
df_ndvi['value_diff'] = df_ndvi.groupby(['system:index', 'class', 'sub_class'])['value'].diff()
# Find rows where the absolute difference is > 0.4
spike_mask = df_ndvi['value_diff'].abs() > 0.4
df_ndvi.loc[spike_mask, 'value'] = np.nan
print(f"Set {spike_mask.sum()} NDVI values to NaN due to > 0.4 difference.")
# Recombine the filtered NDVI data with the VV/VH data
df_long = pd.concat([df_ndvi.drop(columns=['value_diff']), df_other])
print("Data loading and preparation complete.")
return df_long.drop(columns=['metric_date'])
def get_common_date_range(df_long: pd.DataFrame) -> (pd.Timestamp, pd.Timestamp):
"""
Finds the common date range (latest start, earliest end)
across all three metrics.
"""
metric_ranges = df_long.groupby('metric')['date'].agg(['min', 'max'])
common_start = metric_ranges['min'].max()
common_end = metric_ranges['max'].min()
print(f"Common date range found: {common_start.date()} to {common_end.date()}")
return common_start, common_end
def process_time_series(df_long: pd.DataFrame, common_start: pd.Timestamp,
common_end: pd.Timestamp,
window: int, poly: int) -> pd.DataFrame:
"""
Interpolates, resamples, and smooths the time series for each point.
"""
print("Starting time-series processing (interpolation, resampling, smoothing)...")
# 1. Define target dates for resampling (1st and 16th of each month)
target_dates_raw = []
# Start from the 1st of the common_start month
current_date = pd.Timestamp(year=common_start.year, month=common_start.month, day=15)
while current_date <= common_end:
# Add the 1st of the month if it's within the common range
if current_date >= common_start:
target_dates_raw.append(current_date)
# Check the 16th of the month
date_16th = pd.Timestamp(year=current_date.year, month=current_date.month, day=15)
# Add the 16th if it's within the common range
if date_16th >= common_start and date_16th <= common_end:
target_dates_raw.append(date_16th)
# Move to the 1st of the next month
current_date = current_date + pd.offsets.MonthBegin(1)
# Ensure dates are sorted and unique
target_dates = pd.DatetimeIndex(sorted(list(set(target_dates_raw))))
print(target_dates)
if len(target_dates) == 0:
print("Warning: No target dates (1st or 16th) fall within the common date range.")
return pd.DataFrame()
print(f"Resampling to {len(target_dates)} target dates (1st & 16th of month)...")
target_days = (target_dates - common_start).days
results = []
grouped_by_point = df_long.groupby(['system:index', 'class', 'sub_class'])
# Unpack both keys
for (point_id, class_val, sub_class_val), point_data in grouped_by_point:
# Process each metric for the current point
for metric in ['NDVI', 'VV', 'VH']:
metric_data_raw = point_data[point_data['metric'] == metric].sort_values('date')
# Drop NaNs (from spike filter) before interpolation ---
metric_data = metric_data_raw.dropna(subset=['value'])
# We need at least 2 points to interpolate
if len(metric_data) < 2:
# Not enough data, fill with NaNs
smoothed_values = [np.nan] * len(target_dates)
else:
# Convert original dates to relative numerical values (days from start)
current_days = (metric_data['date'] - common_start).dt.days
current_values = metric_data['value'].values
# 2. Create interpolation function
f_interp = interp1d(current_days, current_values, kind='linear', fill_value='extrapolate')
# 3. Resample/Interpolate at target dates
resampled_values = f_interp(target_days)
# 4. Smooth the resampled data
# Ensure window size is not larger than the data itself
safe_window = min(window, len(resampled_values))
# Ensure window is odd
if safe_window % 2 == 0:
safe_window -= 1
# Ensure polyorder is less than the (safe) window
safe_poly = min(poly, safe_window - 1)
if safe_window > safe_poly and safe_poly >= 0:
smoothed_values = savgol_filter(resampled_values, safe_window, safe_poly)
else:
smoothed_values = resampled_values
# Store the results
for i, date in enumerate(target_dates):
results.append({
'system:index': point_id,
'class': class_val,
'sub_class': sub_class_val,
'date': date,
'metric': metric,
'value': smoothed_values[i]
})
print("Time-series processing complete.")
return pd.DataFrame(results)
def format_for_output(results_df: pd.DataFrame) -> pd.DataFrame:
"""
Converts the long-format results back to a wide format for saving.
"""
print("Formatting data for output...")
# Create the 'METRIC_DD-MM-YYYY' column name
results_df['col_name'] = results_df['metric'] + '_' + results_df['date'].dt.strftime('%d-%m-%Y')
wide_df = results_df.pivot(index=['system:index', 'class', 'sub_class'], columns='col_name', values='value')
# Clean up for CSV export
wide_df = wide_df.reset_index()
wide_df.columns.name = None
# Sort the columns in ascending dates of NDVI, VV and VH
ndvi_cols = [col for col in wide_df.columns if col.startswith('NDVI_')]
ndvi_cols = sorted(ndvi_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vv_cols = [col for col in wide_df.columns if col.startswith('VV_')]
vv_cols = sorted(vv_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
vh_cols = [col for col in wide_df.columns if col.startswith('VH_')]
vh_cols = sorted(vh_cols, key=lambda x: pd.to_datetime(x.split('_')[1], format='%d-%m-%Y'))
sorted_cols = ndvi_cols + vv_cols + vh_cols
# LOGIC CHANGE: Include 'class' in final output columns
wide_df = wide_df[['system:index', 'class', 'sub_class'] + sorted_cols]
return wide_df
# Ensure window is odd, default to 5 if not
if SAVGOL_WINDOW % 2 == 0:
print(f"Warning: SAVGOL_WINDOW was even ({SAVGOL_WINDOW}), setting to {SAVGOL_WINDOW + 1}.")
SAVGOL_WINDOW += 1
if SAVGOL_POLY >= SAVGOL_WINDOW:
print(f"Warning: SAVGOL_POLY ({SAVGOL_POLY}) must be less than SAVGOL_WINDOW ({SAVGOL_WINDOW}).")
# Adjust polyorder to be valid
SAVGOL_POLY = max(0, SAVGOL_WINDOW - 2) # e.g., if window is 3, poly becomes 1
print(f"Adjusting SAVGOL_POLY to {SAVGOL_POLY}.")
input_path = Path(INPUT_FILE)
output_path = Path(OUTPUT_FILE)
df_long = load_and_prep_data(input_path)
if df_long.empty:
print("Processing stopped due to errors in loading data.")
common_start, common_end = get_common_date_range(df_long)
print(common_end, common_start)
if pd.isna(common_start) or pd.isna(common_end):
print("Error: Could not determine a valid common date range. Check your input data.")
if common_start > common_end:
print(f"Error: Common start date ({common_start.date()}) is after common end date ({common_end.date()}).")
print("This can happen if the time series for different metrics do not overlap.")
results_df = process_time_series(df_long, common_start, common_end,
window=SAVGOL_WINDOW,
poly=SAVGOL_POLY)
if results_df.empty:
print("No results were generated. Check intermediate steps.")
output_df = format_for_output(results_df)
output_df.to_csv(output_path, index=False)
print(f"\nSuccessfully processed data and saved to: {output_path}")
# --- PLOTTING SECTION (Visual Verification) ---
import matplotlib.pyplot as plt
from IPython.display import display
# Load DataFrames for Display
df_initial = pd.read_csv(INPUT_FILE)
df_processed = pd.read_csv(OUTPUT_FILE)
print("Initial Data Head:")
display(df_initial.head())
print("Processed Data Head:")
display(df_processed.head())
# Prepare Long Formats for Plotting
# LOGIC CHANGE: Melt with class
df_initial_long = pd.melt(df_initial, id_vars=['system:index', 'class', 'sub_class'], var_name='metric_date', value_name='value')
df_initial_long = df_initial_long.dropna(subset=['value'])
split_cols_initial = df_initial_long['metric_date'].str.split('_', n=1, expand=True)
df_initial_long['metric'] = split_cols_initial[0]
df_initial_long['date'] = pd.to_datetime(split_cols_initial[1], format='%d-%m-%Y')
df_initial_long = df_initial_long[df_initial_long['metric'].isin(['NDVI', 'VV', 'VH'])].drop(columns=['metric_date'])
# LOGIC CHANGE: Melt with class
df_processed_long = pd.melt(df_processed, id_vars=['system:index', 'class', 'sub_class'], var_name='metric_date', value_name='value')
split_cols_processed = df_processed_long['metric_date'].str.split('_', n=1, expand=True)
df_processed_long['metric'] = split_cols_processed[0]
df_processed_long['date'] = pd.to_datetime(split_cols_processed[1], format='%d-%m-%Y')
df_processed_long = df_processed_long[df_processed_long['metric'].isin(['NDVI', 'VV', 'VH'])].drop(columns=['metric_date'])
# Select a representative point to plot (e.g., the first point in the list)
plot_index = 0
if not df_initial_long.empty:
point_id = df_initial_long['system:index'].unique()[plot_index]
# Get class for title
row = df_initial_long[df_initial_long['system:index'] == point_id].iloc[0]
class_val = row['class']
sub_class_val = row['sub_class']
# Filter for the selected point
df_initial_point = df_initial_long[df_initial_long['system:index'] == point_id]
df_processed_point = df_processed_long[df_processed_long['system:index'] == point_id]
# Plot metrics
metrics = ['NDVI', 'VV', 'VH']
for metric in metrics:
plt.figure(figsize=(12, 6))
df_init = df_initial_point[df_initial_point['metric'] == metric].sort_values('date')
df_proc = df_processed_point[df_processed_point['metric'] == metric].sort_values('date')
plt.plot(df_init['date'], df_init['value'], label='Initial (Raw)', marker='o', linestyle='--', alpha=0.6)
plt.plot(df_proc['date'], df_proc['value'], label='Processed (Savgol)', marker='x', linestyle='-', linewidth=2)
plt.xlabel('Date')
plt.ylabel('Value')
plt.title(f'{metric} Time Series for {point_id} (Class: {class_val} - {sub_class_val})')
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()
else:
print("No data available to plot.")
BELOW IS THE CODE FOR FINDING THE K MEANS FROM THE SYNTHETIC DATASET USING THE SILHOUETTE SCORE AND WCSS FOR ALL THE CLASSES AND THEN MAKING THE NEW SUB CLASS ACCORDING TO THE K MEANS¶
HERE LET SAY WE HAVE 100 WHEAT SAMPLES OUT OF WHICH WE OPTIMALLY SELECT THE K=5 SO HERE WE WILL HAVE THE 5 DIFFERENT NDVI PROFILES OUT OF 100 AND THEN WE WILL LOOP THROUGH ALL 100 SAMPLES THROUGH CLUSTERING TO FIND ITS NEAREST MATCH TO THE CLASS USING THE EUCLIDEAN DISTANCE INTO 5 DIFFERENT SUBCLASS OR DIFFERENT NDVI PROFILE WE GOT¶
THEN LET SAY ALL 5 CLASS HAS 20 SAMPLES OF WHEAT WE WILL TAKE THE AVERAGE AND WE HAVE THE MEDIAN OF ALL 5 CLASS SO THAT WE HAVE THE REFERENCE NDVI PROFILE OF EACH SUB CLASS¶
THEN WE USE THIS MEDIAN OR AVERAGE NDVI PROFILE OF EACH CLASS TO FIND THE EVALUATION METRICS LIKE SSV,SAM,SSD ETC FOR ALL 20 SAMPLES OF EACH CLASS SO THAT WE HAVE THE LOWEST DISTANCE AND HIGHEST DISTANCE WITH RESPECT TO MEDIAN OR AVERAGE NDVI PROFILE OF THAT PARTICULAR CLASS¶
In [ ]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import warnings
warnings.filterwarnings('ignore') # Suppresses KMeans memory warnings for small datasets
INPUT_FILE = '/content/drive/MyDrive/PhD Obj1/Punjab_Main_Updated_With_Manual_Labels.csv'
OUTPUT_CENTROIDS = '/content/drive/MyDrive/PhD Obj1/Temporal_Models_Centroids.csv'
print("Loading dataset for Phase 2: K-Means Clustering...")
df = pd.read_csv(INPUT_FILE)
ndvi_cols = [col for col in df.columns if 'NDVI' in col]
crop_classes = df['Auto_SubClass'].dropna().unique()
print(f"Found {len(crop_classes)} distinct crop classes to cluster.\n")
final_centroids = []
large_classes = df['Auto_SubClass'].value_counts().head(4).index.tolist()
fig, axes = plt.subplots(2, 2, figsize=(16, 10))
axes = axes.flatten()
plot_idx = 0
for crop in crop_classes:
crop_data = df[df['Auto_SubClass'] == crop]
X = crop_data[ndvi_cols].values
n_samples = len(X)
# --- SCENARIO A: Very Small Classes (No clustering needed) ---
if n_samples < 4:
print(f"[{crop}] Only {n_samples} samples. Setting k=1 (Using Mean as Centroid).")
centroid = np.mean(X, axis=0)
centroid_dict = {'Parent_Class': crop, 'Sub_Cluster_ID': f"{crop}_Model_1", 'Samples_in_Cluster': n_samples}
centroid_dict.update({col: val for col, val in zip(ndvi_cols, centroid)})
final_centroids.append(centroid_dict)
continue
# --- SCENARIO B: Large Classes (Elbow & Silhouette) ---
# Max k should not exceed 10, or half the number of samples (whichever is smaller)
max_k = min(10, n_samples // 2)
k_range = range(2, max_k + 1)
wcss = []
sil_scores = []
for k in k_range:
kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
labels = kmeans.fit_predict(X)
wcss.append(kmeans.inertia_)
sil_scores.append(silhouette_score(X, labels))
# Automatically pick the best 'k' based on the highest Silhouette Score
best_k_idx = np.argmax(sil_scores)
best_k = k_range[best_k_idx]
print(f"[{crop}] Samples: {n_samples} | Evaluated k=2 to {max_k} | Optimal k chosen: {best_k}")
# --- PLOTTING (For Top 4 Classes Only) ---
if crop in large_classes and plot_idx < 4:
ax = axes[plot_idx]
color = 'tab:blue'
ax.set_title(f"Metrics for {crop} (n={n_samples})", fontweight='bold')
ax.set_xlabel('Number of Clusters (k)')
ax.set_ylabel('WCSS (Inertia)', color=color)
ax.plot(k_range, wcss, marker='o', color=color, linestyle='--')
ax.tick_params(axis='y', labelcolor=color)
# Instantiate a second axes that shares the same x-axis
ax2 = ax.twinx()
color2 = 'tab:red'
ax2.set_ylabel('Silhouette Score', color=color2)
ax2.plot(k_range, sil_scores, marker='s', color=color2, linewidth=2)
ax2.tick_params(axis='y', labelcolor=color2)
# Draw a line at the chosen optimal k
ax2.axvline(x=best_k, color='green', linestyle=':', label=f'Best k={best_k}')
ax2.legend(loc='upper right')
plot_idx += 1
# --- EXTRACTING THE CENTROIDS (Temporal Models) ---
final_kmeans = KMeans(n_clusters=best_k, random_state=42, n_init=10)
final_labels = final_kmeans.fit_predict(X)
centers = final_kmeans.cluster_centers_
# Count how many samples fell into each cluster
unique, counts = np.unique(final_labels, return_counts=True)
for i, center in enumerate(centers):
centroid_dict = {
'Parent_Class': crop,
'Sub_Cluster_ID': f"{crop}_Model_{i+1}",
'Samples_in_Cluster': counts[i]
}
centroid_dict.update({col: val for col, val in zip(ndvi_cols, center)})
final_centroids.append(centroid_dict)
# Show the plots for thesis documentation
plt.tight_layout()
plt.show()
df_centroids = pd.DataFrame(final_centroids)
df_centroids.to_csv(OUTPUT_CENTROIDS, index=False)
print("\n" + "="*50)
print(f" PHASE 2 COMPLETE: Successfully generated {len(df_centroids)} Temporal Models!")
print(f" Saved Golden Library to: {OUTPUT_CENTROIDS}")
print("="*50)
print(df_centroids[['Parent_Class', 'Sub_Cluster_ID', 'Samples_in_Cluster']].head(15))
BELOW IS THE CODE FOR THE EVALUATION OF THE METRICS OF ALL CENTROIDS OR CLASS OR SUBCLASS WE MADE ABOVE USING THE K MEANS WE FIND ABOVE : ED,SCS,SAM,SID,DTW,DDTW¶
In [ ]:
# !pip install fastdtw
import pandas as pd
import numpy as np
from scipy.spatial.distance import euclidean
from scipy.stats import pearsonr
from fastdtw import fastdtw
import warnings
warnings.filterwarnings('ignore')
INPUT_SAMPLES = '/content/drive/MyDrive/PhD Obj1/Punjab_Main_Updated_With_Manual_Labels.csv'
INPUT_CENTROIDS = '/content/drive/MyDrive/PhD Obj1/Temporal_Models_Centroids.csv'
OUTPUT_METRICS = '/content/drive/MyDrive/PhD Obj1/Phase3_Evaluation_Metrics.csv'
print("Loading data for Phase 3...")
df_samples = pd.read_csv(INPUT_SAMPLES)
df_centroids = pd.read_csv(INPUT_CENTROIDS)
# Ensure no 'Unsure' samples slip through
df_samples = df_samples[df_samples['Auto_SubClass'] != 'Unsure'].copy()
# Isolate the 13 NDVI columns
ndvi_cols = [col for col in df_samples.columns if 'NDVI' in col]
def calc_ed(x, y):
return euclidean(x, y)
def calc_scs(x, y):
if np.std(x) == 0 or np.std(y) == 0: return 0.0
corr, _ = pearsonr(x, y)
return corr
def calc_ssv(ed, scs):
# Exactly as written in Xu et al. (2019)
return np.sqrt((ed)**2 + (1 - scs)**2)
def calc_sam(x, y):
norm_x, norm_y = np.linalg.norm(x), np.linalg.norm(y)
if norm_x == 0 or norm_y == 0: return np.pi/2
cos_theta = np.clip(np.dot(x, y) / (norm_x * norm_y), -1.0, 1.0)
return np.arccos(cos_theta)
def calc_dtw(x, y):
# FIX: Reshaping the 1D arrays into 2D column vectors so Scipy's euclidean doesn't crash
dist, _ = fastdtw(x.reshape(-1, 1), y.reshape(-1, 1), dist=euclidean)
return dist
def calc_ddtw(x, y):
# FIX: Reshaping the derivatives for DDTW as well
dx, dy = np.diff(x), np.diff(y)
dist, _ = fastdtw(dx.reshape(-1, 1), dy.reshape(-1, 1), dist=euclidean)
return dist
def calc_sid(x, y):
eps = 1e-10
px = (x - np.min(x) + eps)
py = (y - np.min(y) + eps)
px, py = px / np.sum(px), py / np.sum(py)
sid_xy = np.sum(px * np.log(px / py))
sid_yx = np.sum(py * np.log(py / px))
return sid_xy + sid_yx
print(f"Evaluating {len(df_samples)} samples against their class Temporal Models...")
results = []
for index, row in df_samples.iterrows():
true_class = row['Auto_SubClass']
x_i = row[ndvi_cols].values.astype(float)
# Grab the temporal models that belong to this specific crop
class_centroids = df_centroids[df_centroids['Parent_Class'] == true_class]
best_ssv = float('inf')
best_centroid_id = None
best_centroid_vec = None
# Find the nearest neighbor centroid using the paper's SSV metric
for _, c_row in class_centroids.iterrows():
x_c = c_row[ndvi_cols].values.astype(float)
ed_val = calc_ed(x_i, x_c)
scs_val = calc_scs(x_i, x_c)
ssv_val = calc_ssv(ed_val, scs_val)
if ssv_val < best_ssv:
best_ssv = ssv_val
best_centroid_id = c_row['Sub_Cluster_ID']
best_centroid_vec = x_c
final_ed = calc_ed(x_i, best_centroid_vec)
final_scs = calc_scs(x_i, best_centroid_vec)
final_sam = calc_sam(x_i, best_centroid_vec)
final_dtw = calc_dtw(x_i, best_centroid_vec)
final_ddtw = calc_ddtw(x_i, best_centroid_vec)
final_sid = calc_sid(x_i, best_centroid_vec)
results.append({
'system:index': row['system:index'],
'True_Class': true_class,
'Assigned_Centroid': best_centroid_id,
'ED': final_ed,
'SCS': final_scs,
'SSV': best_ssv,
'SAM': final_sam,
'DTW': final_dtw,
'DDTW': final_ddtw,
'SID': final_sid
})
df_metrics = pd.DataFrame(results)
df_metrics.to_csv(OUTPUT_METRICS, index=False)
print("\n" + "="*50)
print(f" PHASE 3 COMPLETE: Computed 7 distinct metrics for {len(df_metrics)} samples.")
print(f" Saved Evaluation Metrics to: {OUTPUT_METRICS}")
print("="*50)
print(df_metrics[['system:index', 'Assigned_Centroid', 'SSV', 'DTW']].head(10))
HERE WE WILL ANALYSE THE EVALUATION METRICS WITH RESPECT TO ITS CLASS FOR ALL THE SAMPLES OF THAT CLASS¶
In [ ]:
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
INPUT_METRICS = '/content/drive/MyDrive/PhD Obj1/Phase3_Evaluation_Metrics.csv'
print("Loading Phase 3 Metrics for Visual Inspection...")
df = pd.read_csv(INPUT_METRICS)
top_classes = df['True_Class'].value_counts().head(4).index.tolist()
metrics_to_plot = ['SSV', 'DTW', 'SAM', 'DDTW']
# Create a grid: Rows = Crop Classes, Columns = Metrics
fig, axes = plt.subplots(nrows=len(top_classes), ncols=len(metrics_to_plot),
figsize=(18, 4 * len(top_classes)))
fig.subplots_adjust(hspace=0.4, wspace=0.3)
for i, crop in enumerate(top_classes):
crop_df = df[df['True_Class'] == crop]
for j, metric in enumerate(metrics_to_plot):
ax = axes[i, j] if len(top_classes) > 1 else axes[j]
data = crop_df[metric].dropna()
if len(data) == 0:
continue
# Plot the histogram
ax.hist(data, bins=20, color='skyblue', edgecolor='black', alpha=0.7)
# Calculate Percentiles
p90 = np.percentile(data, 90)
p95 = np.percentile(data, 95)
p98 = np.percentile(data, 98)
# Draw Percentile Lines
ax.axvline(p90, color='green', linestyle='--', linewidth=2, label=f'90th: {p90:.2f}')
ax.axvline(p95, color='orange', linestyle='-', linewidth=2, label=f'95th: {p95:.2f}')
ax.axvline(p98, color='red', linestyle=':', linewidth=2, label=f'98th: {p98:.2f}')
# Formatting
ax.set_title(f"{crop} - {metric}\n(n={len(data)})", fontsize=12, fontweight='bold')
ax.set_xlabel("Distance Score")
ax.set_ylabel("Frequency")
if j == 0:
ax.legend(loc='upper right', fontsize=9)
plt.suptitle("Distance Distributions & Percentile Thresholds for Major Crops",
fontsize=16, fontweight='bold', y=0.92)
plot_path = '/content/drive/MyDrive/PhD Obj1/Distance_Distributions.png'
plt.savefig(plot_path, bbox_inches='tight', dpi=300)
print(f"-> Saved highly detailed distribution plot to: {plot_path}")
plt.show()
BELOW IS THE CODE FOR THE THRESHOLD WE FIND AND MAKE DECISIONS FROM THE ABOVE EVALUATIONN OF ALL THE METRIC WITH RESPECT TO ITS CLASS AND ASSIGN THE MIN SSV TOLERANCE ,MIN DTW TOLERANCE ETC AND ALSO PERCENTILE RULES WE FOUND USING STATISTICALLY BOUNDARIES TO FILTER THE OUTLIERS ETC¶
AFTER GETTING THE THRESHOLD DICTIONARY WE MADE WE TEST IT TO OUR 1 BATCH OF SYNTHETIC DATA TO CHECK WHEATHER OUR DECISION MAKING IS ACCURATE OR NOT CAUSE THE SYNTHETIC DATA IS MANUALLY VERIFIED BY ME SO IT SHOULD GIVE ALMOST 90 % ACCURACY OR ATLEAST ALL SAMPLES SHOULD BE LABLED TO THE CLASS¶
BUT WE GOT THE ANAMOLIES OR UNSURE SAMPLES WHICH DO NOT BELONG TO ANY SMAPLES WHICH MEANS THERE IS INEFFICIENCY IN OUR DECISION . SO FOR THAT REASON WE INTRODUCE NEW RULES BASED APPROACH WHICH IS AFTER THIS CODE¶
In [ ]:
import pandas as pd
import numpy as np
INPUT_METRICS = '/content/drive/MyDrive/PhD Obj1/Phase3_Evaluation_Metrics.csv'
OUTPUT_THRESHOLDS = '/content/drive/MyDrive/PhD Obj1/Threshold_Dictionary.csv'
OUTPUT_ANOMALIES = '/content/drive/MyDrive/PhD Obj1/Diagnostic_Scorecard_Anomalies.csv'
MIN_SSV_TOL = 0.15
MIN_DTW_TOL = 0.30
MIN_SAM_TOL = 0.15
MIN_DDTW_TOL = 0.20
print("Loading Phase 3 metrics for Anomaly Thresholding...")
df = pd.read_csv(INPUT_METRICS)
percentile_rules = {
'Wheat (Late / Double)': 0.98,
'Fodder / Berseem': 0.98,
'Sugarcane': 0.98,
'Wheat (Standard)': 0.95,
'Wheat': 0.95,
'Forest': 0.95,
'Urban': 0.95,
'Barren': 0.95
}
print("Calculating Custom 'Normality Boundaries'...")
thresholds_list = []
clusters = df['Assigned_Centroid'].unique()
for centroid in clusters:
subset = df[df['Assigned_Centroid'] == centroid]
if len(subset) == 0: continue
# Identify the parent class for Phase 5 mapping and rule lookup
parent_class = str(subset['True_Class'].iloc[0]).strip()
p_val = percentile_rules.get(parent_class, 0.95)
# Calculate raw percentiles
raw_ssv = subset['SSV'].quantile(p_val)
raw_dtw = subset['DTW'].quantile(p_val)
raw_ed = subset['ED'].quantile(p_val)
raw_sam = subset['SAM'].quantile(p_val)
raw_ddtw = subset['DDTW'].quantile(p_val)
# BAKE IN LIMITS AND PARENT CLASS FOR PHASE 5
thresholds_list.append({
'Predicted_Class': parent_class, # FIXED: Critical for Phase 5 labeling
'Assigned_Centroid': centroid,
'SSV_Limit': max(raw_ssv, MIN_SSV_TOL), # FIXED: Baked directly into rulebook
'DTW_Limit': max(raw_dtw, MIN_DTW_TOL),
'ED_Limit': raw_ed, # ED magnitude has no logical minimum limit
'SAM_Limit': max(raw_sam, MIN_SAM_TOL),
'DDTW_Limit': max(raw_ddtw, MIN_DDTW_TOL)
})
df_thresholds = pd.DataFrame(thresholds_list)
df_thresholds.to_csv(OUTPUT_THRESHOLDS, index=False)
print(f"-> Saved Bulletproof Threshold Dictionary: {OUTPUT_THRESHOLDS}")
print("\nRunning Hierarchical Gatekeeper and Diagnostics...")
# Merge the pre-baked limits onto the evaluation dataframe
df_eval = pd.merge(df, df_thresholds, on='Assigned_Centroid', how='left')
anomalies_list = []
for index, row in df_eval.iterrows():
# STEP 1: THE GATEKEEPER (SSV Only)
# Uses the pre-baked, protected limits directly from the merged row
if row['SSV'] > row['SSV_Limit']:
# STEP 2: THE DETECTIVES
if row['DTW'] <= row['DTW_Limit']:
diagnostic_label = "Phenological Shift (e.g., Late/Early Sown)"
elif row['SAM'] <= row['SAM_Limit']:
diagnostic_label = "Biomass Variance (e.g., Sparse/Poor Soil/Stress)"
elif row['DDTW'] <= row['DDTW_Limit']:
diagnostic_label = "Abnormal Growth Rate (e.g., Terminal Heat Stress/Early Harvest)"
else:
diagnostic_label = "Complex Anomaly / Mixed Pixel / Weed Patch"
# FIXED: Added ED to the scorecard for full analytical transparency
anomalies_list.append({
'system:index': row['system:index'],
'True_Class': row['True_Class'],
'Assigned_Centroid': row['Assigned_Centroid'],
'Diagnosis': diagnostic_label,
'SSV_Score': round(row['SSV'], 3),
'SSV_Limit': round(row['SSV_Limit'], 3),
'ED_Score': round(row['ED'], 3), # ADDED
'ED_Limit': round(row['ED_Limit'], 3), # ADDED
'DTW_Score': round(row['DTW'], 3),
'DTW_Limit': round(row['DTW_Limit'], 3),
'SAM_Score': round(row['SAM'], 3),
'SAM_Limit': round(row['SAM_Limit'], 3),
'DDTW_Score': round(row['DDTW'], 3),
'DDTW_Limit': round(row['DDTW_Limit'], 3)
})
df_anomalies = pd.DataFrame(anomalies_list)
print("\n" + "="*50)
print(f" PHASE 4 COMPLETE: Evaluated {len(df)} training samples.")
if not df_anomalies.empty:
df_anomalies.to_csv(OUTPUT_ANOMALIES, index=False)
print(f" Caught {len(df_anomalies)} true anomalies in the training set.")
print("-" * 50)
print(df_anomalies['Diagnosis'].value_counts().to_string())
else:
print(" 0 anomalies caught. Your training data is incredibly clean!")
print("="*50)
3 LOGIC BASED RULES¶
Step 1: The Fast Scan¶
When an unknown pixel arrives, the engine compares its 13-timestep NDVI curve to every single model or class in our Golden Library . It calculates the Euclidean Distance (ED) (magnitude) and the Spatial Correlation (SCS) (shape). It combines them into the SSV Score. The centroid or class with the lowest SSV is selected as the initial Predicted Class.
Step 2: The Gatekeeper¶
The code looks inside the Threshold_Dictionary.csv. If the unlabeled pixel's SSV score is lower than the legal limit this is lower than the actual threshold we find from our labeled dataset or synnthetic dataset of that particular class for that specific crop, the Gatekeeper accepts it immediately as a "Perfect Match
If the SSV score is higher than the legal limit, it would just classify it wrong or drop it. our engine sends it to the 3 rules based approach , which uses three specialized "rules" to determine if the math was fooled¶
IF THE BEST SSV IS 0.25 OF THAT¶
1 . OVERLAP : IF THE OVERLAP PCT >= OVERLAP PASS PCT¶
HERE IT MEANS THAT WE FIND THE ABSOLUTE DIFFERENCE BETWEEN THE SAMPLE AND THE MEDIAN/AVERAGE/CENTROID OF THAT CLASS
HERE WE HAVE THE 13 TIME FRAMES THIS IS FROM OCT 15 TO APRIL 15 15 DAYS INNTERVAL SO HERE AT EVERY TIME FRAMES WE MEASURE THE ABSOLUTE DIFFERENCE BETWEEN THE CENTROID AND THE SAMPLE IF THE DISTANCE IS <0.15 WE WILL CONSIDEER THAT POINTS NOW OUT OF 13 TIME FRAMES WE MUST HAVE 80 % OF THE 13 UNDER < 0.15 THIS IS 9 OR 10 OR 11 POINTS MUST HAVE < 0.15 DISTANCE WITH RESPECT TO THE CENTROID AND WE WILL CONSIDER THAT SAMPLE INTO THE CLASS¶
2.THE SAM¶
Uses calc_sam. Spectral Angle Mapper (SAM) measures the angle of the vectors, ignoring magnitude. If a field has poor soil, its NDVI will be physically lower (failing SSV), but the shape of the growth remains identical (passing SAM). This Judge pardons it as Sparse/Poor Illumination.
3 rule 3 sos¶
Uses get_sos_index. It finds the exact timestep where the crop emerged (NDVI > 0.35). If the unknown pixel emerged a few weeks later than the centroid, it calculates the sos_shift_magnitude. If the shift is within 5 timesteps (approx 75 days), this Judge pardons it as Late Sown or Early Sown.
If a pixel fails the Gatekeeper AND fails all three Judges, the engine confidently declares it a True_Anomaly (likely a weed patch, a destroyed crop, or a completely different land cover type).¶
(Training Diagnostics)¶
:After generating the Threshold_Dictionary.csv, the code runs a "Sanity Check" on its own training data. If a training pixel fails the SSV limit, the code acts as a detective:If DTW is low then It's a phenological shift (planted on a different date).If SAM is low then It's a biomass issue (sparse crop or poor soil).If DDTW is low then It's an abnormal growth rate (heat stress).It exports this to an Anomalies Scorecard so we can see exactly how clean our training data really is.
In [ ]:
import pandas as pd
import numpy as np
from scipy.spatial.distance import euclidean
from scipy.stats import pearsonr
import time
import warnings
warnings.filterwarnings('ignore')
# ==========================================
# 1. CONFIGURATION
# ==========================================
INPUT_UNLABELED = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_03_ReadyForAI.csv'
INPUT_CENTROIDS = '/content/drive/MyDrive/PhD Obj1/Temporal_Models_Centroids_V2.csv'
INPUT_RULEBOOK = '/content/drive/MyDrive/PhD Obj1/Threshold_Dictionary.csv'
OUTPUT_LABELED = '/content/drive/MyDrive/PhD Obj1/Batches/Batch_03_Final_Classified.csv'
# Biological Failsafes for the Appeals Court
OVERLAP_TOLERANCE = 0.15 # NDVI points within +/- 0.15 are considered matching
OVERLAP_PASS_PCT = 0.80 # 80% of the curve must match to pardon fog/clouds
FALLBACK_LIMIT = 0.15 # Fallback limit for newly injected V2 centroids
MAX_BIOLOGICAL_SHIFT = 5 # Expanded to 75 days to accommodate extreme late Jan sowings
print("Loading the Smart Classification Engine...")
df_unlabeled = pd.read_csv(INPUT_UNLABELED)
df_centroids = pd.read_csv(INPUT_CENTROIDS)
df_rulebook = pd.read_csv(INPUT_RULEBOOK)
ndvi_cols = [col for col in df_unlabeled.columns if 'NDVI' in col]
# ==========================================
# 2. METRIC MATHEMATICS
# ==========================================
def calc_ed(x, y): return euclidean(x, y)
def calc_scs(x, y):
if np.std(x) == 0 or np.std(y) == 0: return 0.0
corr, _ = pearsonr(x, y)
return corr
def calc_ssv(ed, scs): return np.sqrt((ed)**2 + (1 - scs)**2)
def calc_sam(x, y):
norm_x, norm_y = np.linalg.norm(x), np.linalg.norm(y)
if norm_x == 0 or norm_y == 0: return np.pi/2
cos_theta = np.clip(np.dot(x, y) / (norm_x * norm_y), -1.0, 1.0)
return np.arccos(cos_theta)
def calc_overlap(x, y):
matches = sum(abs(x - y) <= OVERLAP_TOLERANCE)
return matches / len(x)
def get_sos_index(curve):
# Returns -1 if it never crosses 0.35 (e.g., Water, Barren)
for i, val in enumerate(curve):
if val >= 0.35: return i
return -1
# ==========================================
# 3. THE SMART CLASSIFICATION ENGINE
# ==========================================
print(f"Deploying engine on {len(df_unlabeled)} pixels. Scanning library...")
start_time = time.time()
results = []
for index, row in df_unlabeled.iterrows():
x_unknown = row[ndvi_cols].values.astype(float)
# -----------------------------------------------------
# THE FAST SCAN (Find Nearest Neighbor)
# -----------------------------------------------------
best_ssv = float('inf')
best_centroid_id, predicted_class, best_centroid_vec = None, None, None
for _, c_row in df_centroids.iterrows():
x_c = c_row[ndvi_cols].values.astype(float)
ed_val = calc_ed(x_unknown, x_c)
scs_val = calc_scs(x_unknown, x_c)
ssv_val = calc_ssv(ed_val, scs_val)
if ssv_val < best_ssv:
best_ssv = ssv_val
best_centroid_id = c_row['Sub_Cluster_ID']
best_centroid_vec = x_c
predicted_class = c_row['Parent_Class']
# -----------------------------------------------------
# THE GATEKEEPER
# -----------------------------------------------------
rule = df_rulebook[df_rulebook['Assigned_Centroid'] == best_centroid_id]
if rule.empty:
ssv_limit, sam_limit = FALLBACK_LIMIT, FALLBACK_LIMIT
else:
ssv_limit = rule.iloc[0]['SSV_Limit']
sam_limit = rule.iloc[0]['SAM_Limit']
final_label = predicted_class
diagnostic_note = "Perfect Match"
# -----------------------------------------------------
# THE APPEALS COURT (Only triggered if Gatekeeper fails)
# -----------------------------------------------------
if best_ssv > ssv_limit:
overlap_pct = calc_overlap(x_unknown, best_centroid_vec)
sam_score = calc_sam(x_unknown, best_centroid_vec)
sos_sample = get_sos_index(x_unknown)
sos_centroid = get_sos_index(best_centroid_vec)
# JUDGE 2: Band Overlap (The Fog/Cloud Expert)
if overlap_pct >= OVERLAP_PASS_PCT:
final_label = f"{predicted_class} (Atmospheric Artifact)"
diagnostic_note = f"Pardoned: Overlap {overlap_pct:.0%}"
# JUDGE 1: SAM (The Illumination/Biomass Expert)
elif sam_score <= sam_limit:
final_label = f"{predicted_class} (Sparse/Poor Illumination)"
diagnostic_note = f"Pardoned: SAM Angle {sam_score:.3f}"
# JUDGE 3: SOS Shift (The Sowing Window Expert)
elif sos_sample != -1 and sos_centroid != -1:
sos_shift_magnitude = abs(sos_sample - sos_centroid)
# Check if shift is within biological reason (1 to 45 days)
if 1 <= sos_shift_magnitude <= MAX_BIOLOGICAL_SHIFT:
direction = "Late Sown" if sos_sample > sos_centroid else "Early Sown"
final_label = f"{predicted_class} ({direction})"
diagnostic_note = f"Pardoned: SOS Shifted {sos_shift_magnitude} steps"
else:
final_label = "True_Anomaly"
diagnostic_note = f"Rejected: Shift too extreme ({sos_shift_magnitude} steps)"
else:
final_label = "True_Anomaly"
diagnostic_note = f"Rejected: Failed Appeals on Non-Vegetated Class"
# -----------------------------------------------------
# LOG RESULTS
# -----------------------------------------------------
# FIXED: Extract class safely for merging
orig_class = row['class'] if 'class' in row else 'Unknown'
results.append({
'system:index': row['system:index'],
'Original_Class': orig_class,
'Final_Assigned_Class': final_label,
'Matched_Centroid': best_centroid_id,
'Diagnostic_Note': diagnostic_note,
'SSV_Score': round(best_ssv, 3)
})
if (index + 1) % 200 == 0:
print(f" ...processed {index + 1} pixels...")
# ==========================================
# 4. EXPORT AND SUMMARY
# ==========================================
df_results = pd.DataFrame(results)
# FIXED: Safely merge using only system:index to prevent crashes
df_final = pd.merge(df_unlabeled, df_results, on='system:index', how='left')
# Reorder columns for readability
cols = df_final.columns.tolist()
meta_cols = ['system:index', 'Original_Class', 'Final_Assigned_Class', 'Matched_Centroid', 'Diagnostic_Note', 'SSV_Score']
sensor_cols = [c for c in cols if c not in meta_cols and c != 'class']
df_final = df_final[meta_cols + sensor_cols]
df_final.to_csv(OUTPUT_LABELED, index=False)
elapsed = round((time.time() - start_time) / 60, 2)
print("\n" + "="*50)
print(f" PHASE 5/6 COMPLETE: Processed {len(df_final)} pixels in {elapsed} minutes.")
print(f" Saved labeled dataset to: {OUTPUT_LABELED}")
print("-" * 50)
print("Classification Breakdown:")
print(df_final['Final_Assigned_Class'].value_counts().to_string())
print("="*50)