DRS4 timelapse files#

[1]:

import tables
import matplotlib.pyplot as plt
import numpy as np
from lstcam_calib.io import get_dataset_keys

HDF5 format#

[2]:

file = "../../../test_data/real/service/PixelCalibration/Cat-A/drs4_timelapse/20200218/pro/drs4_timelapse.Run02005.0000.h5"
hdf5_f=tables.open_file(file, "r")
hdf5_f

[2]:

File(filename=../../../test_data/real/service/PixelCalibration/Cat-A/drs4_timelapse/20200218/pro/drs4_timelapse.Run02005.0000.h5, title=np.str_(''), mode='r', root_uep='/', filters=Filters(complevel=5, complib='blosc:zstd', shuffle=True, bitshuffle=False, fletcher32=True, least_significant_digit=None))
/ (RootGroup) np.str_('')
/r0 (Group) np.str_('')
/r0/service (Group) np.str_('')
/r0/service/timelapse_coefficients (Group) np.str_('')
/r0/service/timelapse_coefficients/tel_001 (Table(np.int64(1),)fletcher32, shuffle, blosc:zstd(5)) np.str_('Storage of TimeLapseCoefficients')
  description := {
  "scale": Float32Col(shape=(np.int64(2),), dflt=np.float32(0.0), pos=0),
  "exponent": Float32Col(shape=(np.int64(2),), dflt=np.float32(0.0), pos=1),
  "t0": Float32Col(shape=(np.int64(2),), dflt=np.float32(0.0), pos=2),
  "chi2": Float32Col(shape=(np.int64(2),), dflt=np.float32(0.0), pos=3),
  "mean_10ms": Float32Col(shape=(np.int64(2),), dflt=np.float32(0.0), pos=4),
  "pixel_batch": Int8Col(shape=(np.int64(2), np.int64(1855)), dflt=np.int8(0), pos=5)}
  byteorder := 'little'
  chunkshape := (np.int64(34),)

Keys#

[3]:

key=get_dataset_keys(file)
key

[3]:

['/r0/service/timelapse_coefficients/tel_001']

Data#

[4]:

# The coefficients are estimated per DRS4 batch type
# The pixel batch is reported in the array "pixel_batch"
# LST-1 has all pixels from batch = 0
# LST-2-4 have pixels from batch 0 and 1

drs4_data = hdf5_f.root.r0.service.timelapse_coefficients.tel_001[0]
drs4_data["pixel_batch"]

[4]:

array([[0, 0, 0, ..., 0, 0, 0],
       [0, 0, 0, ..., 0, 0, 0]], shape=(2, 1855), dtype=int8)

[5]:

# for LST-1 only the batch = 0 coefficients are defined
drs4_data["exponent"]

[5]:

array([0.16026449,        nan], dtype=float32)

[6]:

# other way, with ctapipe
from ctapipe.io import read_table
# read the table
table = read_table(file, key[0])
table

/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

[6]:

Table length=1

scale	exponent	t0	chi2	mean_10ms	pixel_batch
float32[2]	float32[2]	float32[2]	float32[2]	float32[2]	int8[2,1855]
23.292261 .. nan	0.16026449 .. nan	55.0 .. nan	0.41111603 .. nan	7.0728245 .. nan	0 .. 0

Meta-data#

[7]:

hdf5_f.root._v_attrs

[7]:

/._v_attrs (AttributeSet), 41 attributes:
   [CLASS := np.str_('GROUP'),
    CTA ACTIVITY ID := np.str_('061f3055-c634-43d7-830d-869c0b4d1541'),
    CTA ACTIVITY NAME := np.str_('lstcam_calib_create_drs4_dt_file'),
    CTA ACTIVITY SOFTWARE NAME := np.str_('lstcam_calib'),
    CTA ACTIVITY SOFTWARE VERSION := np.str_('0.1.2.dev35+g9572ee0'),
    CTA ACTIVITY START TIME := np.str_('2025-03-03 11:13:13.558'),
    CTA ACTIVITY STOP TIME := np.str_('2025-03-03 11:13:18.422'),
    CTA ACTIVITY TYPE := np.str_('software'),
    CTA CONTACT EMAIL := np.str_('unknown'),
    CTA CONTACT NAME := np.str_('Franca Cassol'),
    CTA CONTACT ORGANIZATION := np.str_('LST Consortium'),
    CTA INSTRUMENT CLASS := np.str_('Camera'),
    CTA INSTRUMENT ID := np.str_('unspecified'),
    CTA INSTRUMENT SITE := np.str_('CTA-North'),
    CTA INSTRUMENT SUBTYPE := np.str_('unspecified'),
    CTA INSTRUMENT TYPE := np.str_('LST'),
    CTA INSTRUMENT VERSION := np.str_('unspecified'),
    CTA PROCESS ID := np.str_(''),
    CTA PROCESS SUBTYPE := np.str_('Calibration'),
    CTA PROCESS TYPE := np.str_('Observation'),
    CTA PRODUCT CREATION TIME := np.str_('2025-03-03 11:13:18.449'),
    CTA PRODUCT DATA ASSOCIATION := np.str_('Telescope'),
    CTA PRODUCT DATA CATEGORY := np.str_('A'),
    CTA PRODUCT DATA LEVELS := np.str_('R1'),
    CTA PRODUCT DATA MODEL NAME := np.str_('Unofficial monitoring R1'),
    CTA PRODUCT DATA MODEL URL := np.str_('unknown'),
    CTA PRODUCT DATA MODEL VERSION := np.str_('1.0'),
    CTA PRODUCT DESCRIPTION := np.str_('DRS4 time lapse coefficients'),
    CTA PRODUCT FORMAT := np.str_(''),
    CTA PRODUCT ID := np.str_('11316ec9-8f69-4f9c-a251-58408cd8ebc5'),
    CTA REFERENCE VERSION := np.str_('1'),
    CTAPIPE_IO_LST_VERSION := np.str_('0.25.1'),
    CTAPIPE_VERSION := np.str_('0.19.3'),
    FILTERS := Filters(complevel=5, complib='blosc:zstd', shuffle=True, bitshuffle=False, fletcher32=True, least_significant_digit=None),
    LSTCAM_CALIB_VERSION := np.str_('0.1.2.dev35+g9572ee0'),
    PROV_LOG := np.str_('/fefs/aswg/workspace/franca.cassol/data/real/monitoring/PixelCalibration/Cat-A_lstcam_calib/drs4_timelapse/20200218/v0.1.2.dev35+g9572ee0/log/drs4_timelapse.Run02005.0000_2025-03-03T11:12:36.provenance.log'),
    PYTABLES_FORMAT_VERSION := np.str_('2.1'),
    RUN_START := np.str_('2020-02-18 19:43:25.000'),
    TEL_ID := np.int64(1),
    TITLE := np.str_(''),
    VERSION := np.str_('1.0')]

Fits format#

[8]:

from astropy.io import fits

file = "../../../test_data/real/service/PixelCalibration/Cat-A/drs4_timelapse/20200218/pro/drs4_timelapse.Run02005.0000.fits"
fits_f = fits.open(file)

HDU#

[9]:

fits_f.info()

Filename: ../../../test_data/real/service/PixelCalibration/Cat-A/drs4_timelapse/20200218/pro/drs4_timelapse.Run02005.0000.fits
No.    Name      Ver    Type      Cards   Dimensions   Format
  0  PRIMARY       1 PrimaryHDU      40   ()
  1  SCALE         1 ImageHDU         7   (2,)   float32
  2  EXPONENT      1 ImageHDU         7   (2,)   float32
  3  T0            1 ImageHDU         7   (2,)   float32
  4  PIXEL_BATCH    1 ImageHDU        10   (1855, 2)   uint8 (rescales to int8)

Data#

[10]:

# The coefficients are estimated per DRS4 batch type
# The pixel batch is reported in the array "pixel_batch"
# LST-1 has all pixels from batch = 0
# LST-2-4 have pixels from batch 0 and 1

fits_f["PIXEL_BATCH"].data

[10]:

array([[0, 0, 0, ..., 0, 0, 0],
       [0, 0, 0, ..., 0, 0, 0]], shape=(2, 1855), dtype=int8)

[11]:

# for LST-1 only the batch = 0 coefficients are defined
drs4_data["exponent"]

[11]:

array([0.16026449,        nan], dtype=float32)

Meta-data#

[12]:

fits_f["PRIMARY"].header

[12]:

SIMPLE  =                    T / conforms to FITS standard
BITPIX  =                    8 / array data type
NAXIS   =                    0 / number of array dimensions
EXTEND  =                    T
HIERARCH CTA REFERENCE VERSION = '1       '
HIERARCH CTA CONTACT EMAIL = 'unknown '
HIERARCH CTA CONTACT NAME = 'Franca Cassol'
HIERARCH CTA CONTACT ORGANIZATION = 'LST Consortium'
HIERARCH CTA PRODUCT CREATION TIME = '2025-03-03 12:52:59.835'
HIERARCH CTA PRODUCT DATA ASSOCIATION = 'Telescope'
HIERARCH CTA PRODUCT DATA CATEGORY = 'A       '
HIERARCH CTA PRODUCT DATA LEVELS = 'R1      '
HIERARCH CTA PRODUCT DATA MODEL NAME = 'Unofficial monitoring R1'
HIERARCH CTA PRODUCT DATA MODEL URL = 'unknown '
HIERARCH CTA PRODUCT DATA MODEL VERSION = '1.0     '
HIERARCH CTA PRODUCT DESCRIPTION = 'DRS4 time lapse coefficients'
HIERARCH CTA PRODUCT FORMAT = ''
HIERARCH CTA PRODUCT ID = 'd2d02b3a-2460-4a7d-a6b6-4ed6db5e75ff'
HIERARCH CTA PROCESS ID = ''
HIERARCH CTA PROCESS SUBTYPE = 'Calibration'
HIERARCH CTA PROCESS TYPE = 'Observation'
HIERARCH CTA ACTIVITY ID = 'fdbc1f42-5d2b-46d4-aae2-4a0f968ea2f6'
HIERARCH CTA ACTIVITY NAME = 'lstcam_calib_create_drs4_dt_file'
HIERARCH CTA ACTIVITY SOFTWARE NAME = 'lstcam_calib'
HIERARCH CTA ACTIVITY SOFTWARE VERSION = '0.1.2.dev35+g9572ee0'
HIERARCH CTA ACTIVITY START TIME = '2025-03-03 12:52:57.049'
HIERARCH CTA ACTIVITY STOP TIME = '2025-03-03 12:52:59.830'
HIERARCH CTA ACTIVITY TYPE = 'software'
HIERARCH CTA INSTRUMENT CLASS = 'Camera  '
HIERARCH CTA INSTRUMENT ID = 'unspecified'
HIERARCH CTA INSTRUMENT SITE = 'CTA-North'
HIERARCH CTA INSTRUMENT SUBTYPE = 'unspecified'
HIERARCH CTA INSTRUMENT TYPE = 'LST     '
HIERARCH CTA INSTRUMENT VERSION = 'unspecified'
HIERARCH LSTCAM_CALIB_VERSION = '0.1.2.dev35+g9572ee0'
HIERARCH CTAPIPE_VERSION = '0.19.3  '
HIERARCH CTAPIPE_IO_LST_VERSION = '0.25.1  '
TEL_ID  =                    1
PROV_LOG= '/fefs/aswg/workspace/franca.cassol/data/real/monitoring/PixelCalibr&'
CONTINUE  'ation/Cat-A_lstcam_calib/drs4_timelapse/20200218/v0.1.2.dev35+g9572&'
CONTINUE  'ee0/log/drs4_timelapse.Run02005.0000_2025-03-03T12:52:50.provenance&'
CONTINUE  '.log'
HIERARCH RUN_START = '2020-02-18 19:43:25.000'

[13]:

fits_f["SCALE"].header

[13]:

XTENSION= 'IMAGE   '           / Image extension
BITPIX  =                  -32 / array data type
NAXIS   =                    1 / number of array dimensions
NAXIS1  =                    2
PCOUNT  =                    0 / number of parameters
GCOUNT  =                    1 / number of groups
EXTNAME = 'SCALE   '           / extension name

[ ]: