Hi Gilbert,
> NWS says the data is still going out fine.
I haven't checked in a little bit but this has been an on/off problem for
something like a year now. The one minute aggregations, level2, that data
is still going out last I saw. But they stopped sending the five minute
aggregations, level3, which is what your site uses. The argument was end
sites (i.e. WFOs) can make their own five-minute aggregations from the
one-minute data. This is something AWIPS can do, and since that is the
intended audience... well this is all working as intended then right??
I'm including a python script that can do this aggregation, you'll need to
make a couple edits for your environment. IIRC I put this on your system
and was about to turn it on when I saw they reinstated flowing level3 data,
so I left it there disabled in case this happened again. I forget exactly
what state that's in, but you might just need to enable a pqact if you're
lucky. Look around to see if I already put this script someplace there for
you.
best,
-Mike
On Mon, Apr 27, 2026 at 10:38 AM Sebenste, Gilbert <sebensteg@xxxxxxx>
wrote:
> Good morning everyone,
>
> About a week ago, I suddenly lost our GLM derived flash data for no
> apparent reason. Single strikes are coming in and plotting fine, but
> derived data such as Total Optical Energy (TOE), etc suddenly disappeared
> on my end. NWS says the data is still going out fine.
>
>
> Until I can figure this out on my end, I decided that maybe I should just
> get it from the UNIDATA ADDE server
> for now. I tried to do this:
>
> dsserve.k ADD RTGOESR/GLMEVENT GLMN TYPE=POINT
> INFO=/home/mcidas/data/GLM_EVENT.cfg ADDE.UNIDATA.UCAR.EDU
>
> But it gives me this error:
> dsserve.k: You must enter a min and max file pair with this dataset.
>
> dsserve.k: done
>
> Can anyone tell me what am I doing wrong?
>
> Thank you!
>
>
>
> Gilbert Sebenste
>
> Meteorology Support Analyst
>
>
> _________________________________________________________
> NOTE: All exchanges posted to NSF Unidata maintained email lists are
> made publicly available through the web. Users who post to any of the
> lists we maintain are reminded to remove any personal information that
> they do not want to be made public.
>
> NSF Unidata mcidas-x Mailing List
> (mcidas-x@xxxxxxxxxxxxxxxx)
> For list information, to unsubscribe, or change your membership options,
> visit: https://mailinglists.unidata.ucar.edu/listinfo/mcidas-x/
>
--
- Mike Zuranski
#!/home/ldm/.conda/envs/dev/bin/python
"""
Minimal GLM L2→L3 aggregation (5 min L3 from 5 x 1-min L2).
Mike Zuranski - 2026
"""
import pathlib
import re
from datetime import datetime, timedelta
import xarray as xr
import numpy as np
from datetime import datetime as dt
L2_DIR = pathlib.Path("/home/localdata/satellite/glm/GLMISatSS/level2")
L3_DIR = pathlib.Path("/home/localdata/satellite/glm/GLMISatSS/level3")
L3_DIR.mkdir(parents=True, exist_ok=True)
def extract_time(path):
"""Extract start time from OR_GLM-L2 filename."""
m = re.search(r"s(\d{13})", path.stem)
if not m:
raise ValueError(f"Cannot parse time from {path.name}")
return datetime.strptime(m.group(1), "%Y%j%H%M%S")
def main():
files = sorted(L2_DIR.glob("OR_GLM-L2-GLMF*.nc"))
if not files:
print("No L2 files found")
return
times = [extract_time(f) for f in files]
# Process most recent complete 5-file window
for i in range(len(times) - 1, -1, -1):
t_end = times[i]
t_start = t_end - timedelta(minutes=4)
window = [f for f, t in zip(files, times) if t_start <= t <= t_end]
if len(window) == 5:
aggregate_and_write(window)
break
def aggregate_and_write(window):
"""Aggregate 5 L2 files into 1 L3 file."""
print(f"Aggregating {len(window)} files...")
# Load first file as template
ds_template = xr.open_dataset(window[0], decode_cf=False)
# Initialize accumulators
fed_sum = None
toe_sum = None
mfa_min = None
# Aggregate
for p in window:
ds = xr.open_dataset(p, decode_cf=False)
# FED: sum (counts up)
if fed_sum is None:
fed_sum = ds["Flash_extent_density"].values.astype(np.uint32)
else:
fed_sum += ds["Flash_extent_density"].values.astype(np.uint32)
# TOE: sum (cumulative energy)
if toe_sum is None:
toe_sum = ds["Total_Optical_energy"].values.astype(np.uint32)
else:
toe_sum += ds["Total_Optical_energy"].values.astype(np.uint32)
# MFA: keep minimum
if mfa_min is None:
mfa_min = ds["Minimum_flash_area"].values.astype(np.uint32)
else:
mfa_min = np.minimum(mfa_min,
ds["Minimum_flash_area"].values.astype(np.uint32))
ds.close()
# Convert back to int16
fed_out = np.clip(fed_sum, 0, 65535).astype(np.uint16)
toe_out = np.clip(toe_sum, 0, 65535).astype(np.uint16)
mfa_out = np.clip(mfa_min, 0, 65535).astype(np.uint16)
# print(f"FED min-max: {fed_out.min()}–{fed_out.max()},
nonzero={np.count_nonzero(fed_out)}")
# print(f"TOE min-max: {toe_out.min()}–{toe_out.max()},
nonzero={np.count_nonzero(toe_out)}")
# print(f"MFA min-max: {mfa_out.min()}–{mfa_out.max()},
nonzero={np.count_nonzero(mfa_out)}")
# Get center time and attributes
ds_mid = xr.open_dataset(window[len(window) // 2], decode_cf=False)
time_val = ds_mid["time"].values
time_attrs = dict(ds_mid["time"].attrs)
ds_mid.close()
# Derive start/end times from filename tokens (preferred) or fall back to
file time
s_tok = re.search(r"s(\d{13})", window[0].name)
e_tok = re.search(r"e(\d{13})", window[-1].name)
if s_tok:
start_dt = datetime.strptime(s_tok.group(1), "%Y%j%H%M%S")
else:
ds_start = xr.open_dataset(window[0], decode_cf=False)
start_dt = dt(2017, 1, 1) +
timedelta(seconds=int(ds_start["time"].values))
ds_start.close()
if e_tok:
end_dt = datetime.strptime(e_tok.group(1), "%Y%j%H%M%S")
else:
ds_end = xr.open_dataset(window[-1], decode_cf=False)
end_dt = dt(2017, 1, 1) + timedelta(seconds=int(ds_end["time"].values))
ds_end.close()
# Extract attributes only for the 3 vars we keep
fed_attrs = dict(ds_template["Flash_extent_density"].attrs)
toe_attrs = dict(ds_template["Total_Optical_energy"].attrs)
mfa_attrs = dict(ds_template["Minimum_flash_area"].attrs)
proj_attrs = dict(ds_template["goes_imager_projection"].attrs)
y_attrs = dict(ds_template.coords["y"].attrs)
x_attrs = dict(ds_template.coords["x"].attrs)
# Update FED valid_max for 5-minute sum (5 files × max 50 = 250)
# fed_attrs["valid_max"] = np.int16(50)
fed_attrs.pop("valid_min", None)
fed_attrs.pop("valid_max", None)
fed_attrs["standard_name"] = "Flash Extent Density Five Minute Accumulation"
fed_attrs["long_name"] = "GLM Flash Extent Density Five Minute Accumulation"
# Build minimal L3 dataset with ONLY the 3 gridded variables
# Use np.int32(0) for projection to ensure 32-bit int (not 64-bit)
out = xr.Dataset(
coords={
"y": (("y",), ds_template.coords["y"].values, y_attrs),
"x": (("x",), ds_template.coords["x"].values, x_attrs),
},
data_vars={
"time": ((), time_val, time_attrs),
"Flash_extent_density": (("y", "x"), fed_out, fed_attrs),
"Total_Optical_energy": (("y", "x"), toe_out, toe_attrs),
"Minimum_flash_area": (("y", "x"), mfa_out, mfa_attrs),
"goes_imager_projection": ((), np.int32(0), proj_attrs),
},
attrs=ds_template.attrs,
)
# Rename variables before saving
out = out.rename({
"Flash_extent_density": "Flash_extent_density_w5u1",
"Total_Optical_energy": "Total_Optical_energy_w5u1",
"Minimum_flash_area": "Minimum_flash_area_w5u1",
})
# Update metadata with proper time coverage
out.attrs["title"] = "GLM L3 Lightning Detection Gridded Product"
# Set time coverage attributes
out.attrs["time_coverage_start"] = start_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
out.attrs["time_coverage_end"] = end_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
# Build filename (L2 > L3)
base = re.sub(r"_s\d+.*", "", window[0].name)
base = base.replace("-L2-", "-L3-")
s_tok = re.search(r"s(\d+)", window[0].name)
e_tok = re.search(r"e(\d+)", window[-1].name)
c_tok = re.search(r"c(\d+)", window[len(window) // 2].name)
s = s_tok.group(1)
e = e_tok.group(1)
c = c_tok.group(1)
filename = f"{base}_s{s}_e{e}_c{c}.nc"
# Encoding: zlib compression for gridded vars, correct dtypes for all
encoding = {
"Flash_extent_density_w5u1": {"dtype": "int16", "zlib": True,
"complevel": 6},
"Total_Optical_energy_w5u1": {"dtype": "int16", "zlib": True,
"complevel": 6},
"Minimum_flash_area_w5u1": {"dtype": "int16", "zlib": True,
"complevel": 6},
"y": {"dtype": "int16"},
"x": {"dtype": "int16"},
"time": {"dtype": "int32"},
"goes_imager_projection": {"dtype": "int32"},
}
# Write
out_path = L3_DIR / filename
out.to_netcdf(out_path, encoding=encoding)
# file_size = out_path.stat().st_size / 1024
# print(f"Wrote {out_path}")
# print(f" Time coverage: {out.attrs['time_coverage_start']} to
{out.attrs['time_coverage_end']}")
# print(f" File size: {file_size:.1f} KB")
ds_template.close()
if __name__ == "__main__":
main()