Calibration files#

[1]:

import tables
import matplotlib.pyplot as plt
import numpy as np
from lstcam_calib.io import get_dataset_keys

HDF5 format#

[2]:

file = "../../../test_data/real/service/PixelCalibration/Cat-A/calibration//20200218/pro/calibration_filters_52.Run02006.0000.h5"
hdf5_f=tables.open_file(file, "r")
hdf5_f

[2]:

File(filename=../../../test_data/real/service/PixelCalibration/Cat-A/calibration//20200218/pro/calibration_filters_52.Run02006.0000.h5, title=np.str_(''), mode='r', root_uep='/', filters=Filters(complevel=5, complib='blosc:zstd', shuffle=True, bitshuffle=False, fletcher32=True, least_significant_digit=None))
/ (RootGroup) np.str_('')
/tel_1 (Group) np.str_('')
/tel_1/calibration (Table(np.int64(1),)fletcher32, shuffle, blosc:zstd(5)) np.str_('Storage of WaveformCalibrationContainer')
  description := {
  "time": Float64Col(shape=(), dflt=np.float64(0.0), pos=0),
  "time_min": Float64Col(shape=(), dflt=np.float64(0.0), pos=1),
  "time_max": Float64Col(shape=(), dflt=np.float64(0.0), pos=2),
  "dc_to_pe": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=3),
  "pedestal_per_sample": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=4),
  "time_correction": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=5),
  "n_pe": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=6),
  "unusable_pixels": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=7)}
  byteorder := 'little'
  chunkshape := (np.int64(4),)
/tel_1/flatfield (Table(np.int64(1),)fletcher32, shuffle, blosc:zstd(5)) np.str_('Storage of FlatFieldContainer')
  description := {
  "sample_time": Float64Col(shape=(), dflt=np.float64(0.0), pos=0),
  "sample_time_min": Float64Col(shape=(), dflt=np.float64(0.0), pos=1),
  "sample_time_max": Float64Col(shape=(), dflt=np.float64(0.0), pos=2),
  "n_events": Int64Col(shape=(), dflt=np.int64(0), pos=3),
  "charge_mean": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=4),
  "charge_median": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=5),
  "charge_std": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=6),
  "time_mean": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=7),
  "time_median": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=8),
  "time_std": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=9),
  "relative_gain_mean": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=10),
  "relative_gain_median": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=11),
  "relative_gain_std": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=12),
  "relative_time_median": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=13),
  "charge_median_outliers": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=14),
  "charge_std_outliers": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=15),
  "time_median_outliers": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=16)}
  byteorder := 'little'
  chunkshape := (np.int64(1),)
/tel_1/pedestal (Table(np.int64(1),)fletcher32, shuffle, blosc:zstd(5)) np.str_('Storage of PedestalContainer')
  description := {
  "n_events": Int64Col(shape=(), dflt=np.int64(0), pos=0),
  "sample_time": Float64Col(shape=(), dflt=np.float64(0.0), pos=1),
  "sample_time_min": Float64Col(shape=(), dflt=np.float64(0.0), pos=2),
  "sample_time_max": Float64Col(shape=(), dflt=np.float64(0.0), pos=3),
  "charge_mean": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=4),
  "charge_median": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=5),
  "charge_std": Float64Col(shape=(np.int64(2), np.int64(1855)), dflt=np.float64(0.0), pos=6),
  "charge_median_outliers": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=7),
  "charge_std_outliers": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=8)}
  byteorder := 'little'
  chunkshape := (np.int64(2),)
/tel_1/pixel_status (Table(np.int64(1),)fletcher32, shuffle, blosc:zstd(5)) np.str_('Storage of PixelStatusContainer')
  description := {
  "hardware_failing_pixels": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=0),
  "pedestal_failing_pixels": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=1),
  "flatfield_failing_pixels": BoolCol(shape=(np.int64(2), np.int64(1855)), dflt=np.False_, pos=2)}
  byteorder := 'irrelevant'
  chunkshape := (np.int64(23),)

Keys#

[3]:

keys=get_dataset_keys(file)
keys

[3]:

['/tel_1/calibration',
 '/tel_1/flatfield',
 '/tel_1/pedestal',
 '/tel_1/pixel_status']

Data#

[4]:

# one can access per array
calibration = hdf5_f.root.tel_1.calibration[0]
calibration["dc_to_pe"]

[4]:

array([[0.01226568, 0.01231556, 0.01246075, ..., 0.01230908, 0.01208743,
        0.01192171],
       [0.2170062 , 0.2185252 , 0.21890534, ..., 0.21310168, 0.21461796,
        0.21522383]], shape=(2, 1855))

[5]:

# or use read_table for ctapipe

from ctapipe.io import read_table

table = read_table(file, keys[0])
table

/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
  from .autonotebook import tqdm as notebook_tqdm

[5]:

Table length=1

time	time_min	time_max	dc_to_pe	pedestal_per_sample	time_correction	n_pe	unusable_pixels
s	s	s			ns
float64	float64	float64	float64[2,1855]	float64[2,1855]	float64[2,1855]	float64[2,1855]	bool[2,1855]
1582057036.6918879	1582057031.624238	1582057041.7595375	0.012265676535825266 .. 0.2152238287374495	-1.7157643636067708 .. 0.334843377272288	-1.0010204315185547 .. 0.3804140090942383	92.80542577714914 .. 76.78684778771262	False .. False

[6]:

# or fill the ctapipe monitoring container with a lstcam_calib reading function

from lstcam_calib.io.calibration import read_calibration_file

monitoring = read_calibration_file(file)

monitoring

[6]:

ctapipe.containers.MonitoringCameraContainer:
                   flatfield.*: Data from flat-field event distributions with
                                default None
                    pedestal.*: Data from pedestal event distributions with
                                default None
                pixel_status.*: Container for masks with pixel status with
                                default None
                 calibration.*: Container for calibration coefficients with
                                default None

[7]:

monitoring.calibration

[7]:

ctapipe.containers.WaveformCalibrationContainer:
                          time: Time associated to the calibration event with
                                default nan s [s]
                      time_min: Earliest time of validity for the calibration
                                event with default nan s [s]
                      time_max: Latest time of validity for the calibration
                                event with default nan s [s]
                      dc_to_pe: np array of (digital count) to (photon electron)
                                coefficients (n_chan, n_pix) with default None
           pedestal_per_sample: np array of average pedestal value per sample
                                (digital count) (n_chan, n_pix) with default
                                None
               time_correction: np array of time correction values (n_chan,
                                n_pix) with default None
                          n_pe: np array of photo-electrons in calibration
                                signal (n_chan, n_pix) with default None
               unusable_pixels: Boolean np array of final calibration data
                                analysis, True = failing pixels (n_chan, n_pix)
                                with default None

Meta-data#

[8]:

hdf5_f.root._v_attrs
# or
monitoring.meta

[8]:

{'CTA REFERENCE VERSION': '1', 'CTA CONTACT EMAIL': np.str_('unknown'), 'CTA CONTACT NAME': np.str_('Franca Cassol'), 'CTA CONTACT ORGANIZATION': np.str_('LST Consortium'), 'CTA PRODUCT CREATION TIME': '2024-10-14 10:29:10.958', 'CTA PRODUCT DATA ASSOCIATION': np.str_('Telescope'), 'CTA PRODUCT DATA CATEGORY': np.str_('A'), 'CTA PRODUCT DATA LEVELS': 'R1', 'CTA PRODUCT DATA MODEL NAME': np.str_('Unofficial monitoring R1'), 'CTA PRODUCT DATA MODEL URL': np.str_('unknown'), 'CTA PRODUCT DATA MODEL VERSION': np.str_('1.0'), 'CTA PRODUCT DESCRIPTION': np.str_('Cat-A pixel calibration coefficients'), 'CTA PRODUCT FORMAT': np.str_(''), 'CTA PRODUCT ID': np.str_('27d02f9a-1107-4b10-a85d-0ddd1a45c5e2'), 'CTA PROCESS ID': np.str_(''), 'CTA PROCESS SUBTYPE': np.str_('Calibration'), 'CTA PROCESS TYPE': np.str_('Observation'), 'CTA ACTIVITY ID': np.str_('21bd163c-f9f9-4d99-9488-a37f44ce9471'), 'CTA ACTIVITY NAME': np.str_('CalibrationWriter'), 'CTA ACTIVITY SOFTWARE NAME': np.str_('lstcam_calib'), 'CTA ACTIVITY SOFTWARE VERSION': np.str_('0.1.dev365+g4ed7d35'), 'CTA ACTIVITY START TIME': '2024-10-14 10:20:48.643', 'CTA ACTIVITY STOP TIME': '2024-10-14 10:29:10.416', 'CTA ACTIVITY TYPE': np.str_('software'), 'CTA INSTRUMENT CLASS': np.str_('Camera'), 'CTA INSTRUMENT ID': np.str_('unspecified'), 'CTA INSTRUMENT SITE': np.str_('CTA-North'), 'CTA INSTRUMENT SUBTYPE': np.str_('unspecified'), 'CTA INSTRUMENT TYPE': np.str_('LST'), 'CTA INSTRUMENT VERSION': np.str_('unspecified')}

Fits format#

[9]:

from astropy.io import fits

file = "../../../test_data/real/service/PixelCalibration/Cat-A/calibration//20200218/pro/calibration_filters_52.Run02006.0000.fits.gz"
fits_f = fits.open(file)

HDU#

[10]:

fits_f.info()

Filename: ../../../test_data/real/service/PixelCalibration/Cat-A/calibration//20200218/pro/calibration_filters_52.Run02006.0000.fits.gz
No.    Name      Ver    Type      Cards   Dimensions   Format
  0  PRIMARY       1 PrimaryHDU      40   ()
  1  CALIBRATION    1 BinTableHDU     33   1R x 8C   [D, D, D, 3710D, 3710D, 3710D, 3710D, 3710L]
  2  FLATFIELD     1 BinTableHDU     59   1R x 17C   [D, D, D, K, 3710D, 3710D, 3710D, 3710D, 3710D, 3710D, 3710D, 3710D, 3710D, 3710D, 3710L, 3710L, 3710L]
  3  PEDESTAL      1 BinTableHDU     35   1R x 9C   [K, D, D, D, 3710D, 3710D, 3710D, 3710L, 3710L]
  4  PIXEL_STATUS    1 BinTableHDU     18   1R x 3C   [3710L, 3710L, 3710L]

Data#

[11]:

fits_f["CALIBRATION"].data

[11]:

FITS_rec([(1.58205704e+09, 1.58205703e+09, 1.58205704e+09, [[0.01226568, 0.01231556, 0.01246075, ..., 0.01230908, 0.01208743, 0.01192171], [0.2170062 , 0.2185252 , 0.21890534, ..., 0.21310168, 0.21461796, 0.21522383]], [[-1.71576436e+00, -1.41364988e+00, -9.88852223e-01, ..., -1.38245066e+00, -2.20048269e+00, -1.48831669e+00], [-7.16638317e-02, -2.02496668e-01, -2.23805477e-01, ..., -2.83853531e-01, -7.16333439e-02,  3.34843377e-01]], [[-1.00102043e+00, -6.51331902e-01, -1.57318878e+00, ..., -5.15079498e-02,  7.00457573e-01,  2.77062416e-01], [-9.44082260e-01, -9.92441177e-01, -1.92085171e+00, ..., -2.11906433e-03,  5.45783997e-01,  3.80414009e-01]], [[ 92.80542578,  97.5749445 ,  88.26198024, ...,  72.37888967,  77.54240491,  81.67431923], [ 95.36394716, 101.11898778,  86.70701634, ...,  73.48857545,  77.27613535,  76.78684779]], [[False, False, False, ..., False, False, False], [False, False, False, ..., False, False, False]])],
         dtype=(numpy.record, [('time', '>f8'), ('time_min', '>f8'), ('time_max', '>f8'), ('dc_to_pe', '>f8', (2, 1855)), ('pedestal_per_sample', '>f8', (2, 1855)), ('time_correction', '>f8', (2, 1855)), ('n_pe', '>f8', (2, 1855)), ('unusable_pixels', 'i1', (2, 1855))]))

[12]:

# or with astropy tables
from astropy.table import QTable
QTable.read(file, hdu='CALIBRATION')

[12]:

QTable length=1

time	time_min	time_max	dc_to_pe	pedestal_per_sample	time_correction	n_pe	unusable_pixels
s	s	s
float64	float64	float64	float64[2,1855]	float64[2,1855]	float64[2,1855]	float64[2,1855]	bool[2,1855]
1582057036.6918879	1582057031.624238	1582057041.7595375	0.012265676535825266 .. 0.2152238287374495	-1.7157643636067708 .. 0.334843377272288	-1.0010204315185547 .. 0.3804140090942383	92.80542577714914 .. 76.78684778771262	False .. False

[13]:

# or fill the ctapipe monitoring container with a lstcam_calib reading function

from lstcam_calib.io.calibration import read_calibration_file

monitoring = read_calibration_file(file)

monitoring

[13]:

ctapipe.containers.MonitoringCameraContainer:
                   flatfield.*: Data from flat-field event distributions with
                                default None
                    pedestal.*: Data from pedestal event distributions with
                                default None
                pixel_status.*: Container for masks with pixel status with
                                default None
                 calibration.*: Container for calibration coefficients with
                                default None

Meta-data#

[14]:

fits_f["PRIMARY"].header

[14]:

SIMPLE  =                    T / conforms to FITS standard
BITPIX  =                    8 / array data type
NAXIS   =                    0 / number of array dimensions
EXTEND  =                    T
HIERARCH CTA REFERENCE VERSION = '1       '
HIERARCH CTA CONTACT EMAIL = 'unknown '
HIERARCH CTA CONTACT NAME = 'Franca Cassol'
HIERARCH CTA CONTACT ORGANIZATION = 'LST Consortium'
HIERARCH CTA PRODUCT CREATION TIME = '2024-10-14 11:34:18.715'
HIERARCH CTA PRODUCT DATA ASSOCIATION = 'Telescope'
HIERARCH CTA PRODUCT DATA CATEGORY = 'A       '
HIERARCH CTA PRODUCT DATA LEVELS = 'R1      '
HIERARCH CTA PRODUCT DATA MODEL NAME = 'Unofficial monitoring R1'
HIERARCH CTA PRODUCT DATA MODEL URL = 'unknown '
HIERARCH CTA PRODUCT DATA MODEL VERSION = '1.0     '
HIERARCH CTA PRODUCT DESCRIPTION = 'Cat-A pixel calibration coefficients'
HIERARCH CTA PRODUCT FORMAT = ''
HIERARCH CTA PRODUCT ID = '02973365-2507-4410-89ec-001d19a0279a'
HIERARCH CTA PROCESS ID = ''
HIERARCH CTA PROCESS SUBTYPE = 'Calibration'
HIERARCH CTA PROCESS TYPE = 'Observation'
HIERARCH CTA ACTIVITY ID = '83afcd51-4706-4cf4-a6b3-64a84182da50'
HIERARCH CTA ACTIVITY NAME = 'CalibrationWriter'
HIERARCH CTA ACTIVITY SOFTWARE NAME = 'lstcam_calib'
HIERARCH CTA ACTIVITY SOFTWARE VERSION = '0.1.dev365+g4ed7d35'
HIERARCH CTA ACTIVITY START TIME = '2024-10-14 11:26:40.845'
HIERARCH CTA ACTIVITY STOP TIME = '2024-10-14 11:34:18.709'
HIERARCH CTA ACTIVITY TYPE = 'software'
HIERARCH CTA INSTRUMENT CLASS = 'Camera  '
HIERARCH CTA INSTRUMENT ID = 'unspecified'
HIERARCH CTA INSTRUMENT SITE = 'CTA-North'
HIERARCH CTA INSTRUMENT SUBTYPE = 'unspecified'
HIERARCH CTA INSTRUMENT TYPE = 'LST     '
HIERARCH CTA INSTRUMENT VERSION = 'unspecified'
HIERARCH LSTCAM_CALIB_VERSION = '0.1.dev365+g4ed7d35'
HIERARCH CTAPIPE_VERSION = '0.19.3  '
HIERARCH CTAPIPE_IO_LST_VERSION = '0.24.0  '
TEL_ID  =                    1
PROV_LOG= '/fefs/aswg/workspace/franca.cassol/data/real/monitoring/PixelCalibr&'
CONTINUE  'ation/Cat-A_lstcam_calib/calibration/20200218/v0.1.dev365+g4ed7d35/&'
CONTINUE  'log/calibration_filters_52.Run02006.0000_2024-10-14T11:26:32.proven&'
CONTINUE  'ance.log'
HIERARCH RUN_START = '2020-02-18 20:16:59.000'

[15]:

fits_f["CALIBRATION"].header

[15]:

XTENSION= 'BINTABLE'           / binary table extension
BITPIX  =                    8 / array data type
NAXIS   =                    2 / number of array dimensions
NAXIS1  =               122454 / length of dimension 1
NAXIS2  =                    1 / length of dimension 2
PCOUNT  =                    0 / number of group parameters
GCOUNT  =                    1 / number of groups
TFIELDS =                    8 / number of table fields
EXTNAME = 'CALIBRATION'        / extension name
TTYPE1  = 'time    '
TFORM1  = 'D       '
TUNIT1  = 's       '
TTYPE2  = 'time_min'
TFORM2  = 'D       '
TUNIT2  = 's       '
TTYPE3  = 'time_max'
TFORM3  = 'D       '
TUNIT3  = 's       '
TTYPE4  = 'dc_to_pe'
TFORM4  = '3710D   '
TDIM4   = '(1855,2)'
TTYPE5  = 'pedestal_per_sample'
TFORM5  = '3710D   '
TDIM5   = '(1855,2)'
TTYPE6  = 'time_correction'
TFORM6  = '3710D   '
TDIM6   = '(1855,2)'
TTYPE7  = 'n_pe    '
TFORM7  = '3710D   '
TDIM7   = '(1855,2)'
TTYPE8  = 'unusable_pixels'
TFORM8  = '3710L   '
TDIM8   = '(1855,2)'