SHELL := /bin/bash # Use bash syntax # # gstlal offline tutorial # # Author: Duncan Meacher (duncan.meacher@ligo.org) # # This Makefile is designed to be a very basic introduction to the gstlal # workflow. The steps of this Makefile are: # # 1. Produce 2048s of fake data data that contains a single BNS injection with # an early aLIGO PSD. # 2. A simple frames.cache, segments.xml.gz, vetoes.xml.gz, and tisi.xml files # are produced, which are required for running gstlal_insiral. # 3. A very basic template template bank, focused around the injection # parameters, is produced. N_templates < 250. # 4. An SVD bank is then produced from this template bank. # 5. gstlal_inspiral is then run, producing output files containing triggers # and background statistics. # 6. The triggers are then asigned a ranking statistic, and are then clustered # so that only the loudest trigger within a 4s window remains. # 7. The background statistics collected in step 5 are sampled to find the # noise distribution of our ranking statistic. These are then combined across # the entire search. # 8. The noise distributions found in 7 are used to assign false alarm rates # (FARs) and probabilities (FAPs). # 9. Plotting scripts are then run and a summary page is produced. # # This Makefile is entirely self contained. # # NOTE: This Makefile is arranged "backwards" and should be read top to bottom. # # To run, making sure that the correct environment is sourced: # # $ make -f Makefile.offline_tutorial_test # ################## # Runtime outout # ################## # Prints runtime in secoonds of each individual job. # To disable leave blank. #TIME= TIME=time -p ############################## # Data generation parameters # ############################## # The GPS start time to generate data DATASTART = 999999952 # The GPS stop time to generate data DATASTOP = 1000002100 # Sampling rate of fake data SAMPLE_RATE = 4096 ######################### # Triggering parameters # ######################### # The detectors to analyze IFOS = H1 L1 # Minimum number of detecors working to use MIN_IFOS = 2 # The GPS start time for analysis START = 1000000000 # The GPS end time for analysis STOP = 1000002048 # analysis dureation DURATION = $(shell echo $(STOP)-$(START) | bc) # A user tag for the run TAG = gstlal_offline_tutorial # Run number RUN=run_1 # A web directory for output WEBDIR = WEBDIR/$(TAG)/$(START)-$(STOP)-test_dag-$(RUN) #WEBDIR = ~/public_html/gstlal_offline_tutorial/$(TAG)/$(START)-$(STOP)-test_dag-$(RUN) # The number of sub banks in each SVD bank file NUMBANKS = 1 ############################ # Template Bank Parameters # ############################ # Controls the number of templates in each SVD sub bank NUM_SPLIT_TEMPLATES = 250 # Number of bins of chi to uniformly bin templates into NUM_CHI_BINS = 1 # Controls the overlap from sub bank to sub bank - helps mitigate edge effects # in the SVD. Redundant templates will be removed OVERLAP = 50 # The approximant that you wish to filter with for BANK_MCHIRP1 and BANK_MCHIRP2, respectively. APPROXIMANT = 0.0:100.0:TaylorF2 # Template bank parameters LOW_FREQUENCY_CUTOFF = 30.0 HIGH_FREQUENCY_CUTOFF = 1024 #################### # Injection params # #################### # Note: A 1000s range is given so that 100 injections are produced. # The GPS start time for injections INJSTART=1000001020 # The GPS stop time for injections INJSTOP=1000002020 ############################# # additional options, e.g., # ############################# # The minimum number of samples to include in a given time slice SAMPLES_MIN = 2048 # The maximum number of samples to include in the 256 Hz or above time slices SAMPLES_MAX_256 = 2048 ADDITIONAL_DAG_OPTIONS:=--samples-min $(SAMPLES_MIN) --samples-max-256 $(SAMPLES_MAX_256) # whether to disable plotting, pass in ENABLE_PLOTTING=0 to disable ENABLE_PLOTTING ?= 1 ############## # Channels # ############## H1_CHANNEL=FAKE L1_CHANNEL=FAKE CHANNEL_NAMES:=--channel-name=H1=$(H1_CHANNEL) --channel-name=L1=$(L1_CHANNEL) ############ # Workflow # ############ ifeq ($(ENABLE_PLOTTING), 1) all: gstlal_inspiral_summary_page_0002 else all: gstlal_compute_far_from_snr_chisq_histograms_0001 endif plots: # Makes local plots directory mkdir plots @echo "" $(WEBDIR): $(MAKEFILE_LIST) # Make webpage directory and copy files across mkdir -p $(WEBDIR)/OPEN-BOX cp $(MAKEFILE_LIST) $@ @echo "" injections.xml: # Generate injection file containing a single BNS injection $(TIME) lalapps_inspinj \ --m-distr gaussian \ --min-mass1 1.39 \ --max-mass1 1.41 \ --min-mass2 1.39 \ --max-mass2 1.41 \ --min-mtotal 2.79 \ --max-mtotal 2.81 \ --mean-mass1 1.4 \ --mean-mass2 1.4 \ --stdev-mass1 0.01 \ --stdev-mass2 0.01 \ --gps-start-time $(INJSTART) \ --gps-end-time $(INJSTOP) \ --disable-spin \ --d-distr uniform \ --i-distr uniform \ --min-distance 10000 \ --max-distance 15000 \ --waveform TaylorT4threePointFivePN \ --l-distr random \ --f-lower 15. \ --time-step 20 \ --t-distr uniform \ --time-interval 10 \ --seed 12345 \ --output $@ \ --verbose ligolw_no_ilwdchar $@ @echo "" early_aligo_asd.txt: wget https://git.ligo.org/lscsoft/gstlal/-/raw/master/gstlal/share/early_aligo_asd.txt @echo "" H1_psd.xml.gz: early_aligo_asd.txt # Generate H1 PSD used to produce fake data # gstlal_psd_xml_from_asd_txt_0001 gstlal_psd_xml_from_asd_txt --instrument H1 --output $@ early_aligo_asd.txt @echo "" L1_psd.xml.gz: early_aligo_asd.txt # Generate L1 PSD used to produce fake data # gstlal_psd_xml_from_asd_txt_0002 gstlal_psd_xml_from_asd_txt --instrument L1 --output $@ early_aligo_asd.txt @echo "" H-H1_FAKE-10000/H-H1_FAKE-1000000000-512.gwf: H1_psd.xml.gz injections.xml # Generate 2048s of H1 FAKE data with a single injection included # gstlal_fake_frames_0001 $(TIME) gstlal_fake_frames \ --data-source white \ --sample-rate $(SAMPLE_RATE) \ --frame-type H1_FAKE \ --channel-name=H1=$(H1_CHANNEL) \ --gps-start-time $(DATASTART) \ --gps-end-time $(DATASTOP) \ --color-psd H1_psd.xml.gz \ --frames-per-file 512 \ --frame-duration 1 \ --injections injections.xml \ --verbose @echo "" L-L1_FAKE-10000/L-L1_FAKE-1000000000-512.gwf: L1_psd.xml.gz injections.xml # Generate 2048s of L1 FAKE data with a single injection included # gstlal_fake_frames_0002 $(TIME) gstlal_fake_frames \ --data-source white \ --sample-rate $(SAMPLE_RATE) \ --frame-type L1_FAKE \ --channel-name=L1=$(L1_CHANNEL) \ --gps-start-time $(DATASTART) \ --gps-end-time $(DATASTOP) \ --color-psd L1_psd.xml.gz \ --frames-per-file 512 \ --frame-duration 1 \ --injections injections.xml \ --verbose @echo "" frames.cache: H-H1_FAKE-10000/H-H1_FAKE-1000000000-512.gwf L-L1_FAKE-10000/L-L1_FAKE-1000000000-512.gwf # Produce frames cache file # A file that contains full paths to all frame files ls H-H1_FAKE-*/* L-L1_FAKE-*/* | lalapps_path2cache > frames.cache @echo "" segments.xml.gz: # Produce segments file # A files that lists the segments of data that can be analysed echo $(START) $(STOP) | ligolw_segments --insert-from-segwizard=H1=/dev/stdin --name=datasegments --output $@ echo $(START) $(STOP) | ligolw_segments --insert-from-segwizard=L1=/dev/stdin --name=datasegments $@ ligolw_no_ilwdchar $@ @echo "" vetoes.xml.gz: # Produce a dummy veto file @echo "Produce veto file" echo 999999997 999999998 | ligolw_segments --insert-from-segwizard=H1=/dev/stdin --name=vetoes --output $@ echo 999999997 999999998 | ligolw_segments --insert-from-segwizard=L1=/dev/stdin --name=vetoes $@ ligolw_no_ilwdchar $@ @echo "" tisi.xml: # Produce time slides file lalapps_gen_timeslides --instrument=H1=0:0:0 --instrument=L1=0:25.13274:25.13274 $@ @echo "" measured_psd.xml.gz: frames.cache # Job gstlal_reference_psd_0001 $(TIME) gstlal_reference_psd \ --data-source frames \ --frame-cache frames.cache \ --gps-start-time $(START) \ --gps-end-time $(STOP) \ --channel-name=H1=$(H1_CHANNEL) \ --channel-name=L1=$(L1_CHANNEL) \ --sample-rate $(SAMPLE_RATE) \ --write-psd $@ \ --verbose @echo "" psd.png: measured_psd.xml.gz # gstlal_plot_psd_0001 $(TIME) gstlal_plot_psd --output $@ measured_psd.xml.gz @echo "" psd_horizon.png: # gstlal_plot_psd_horizon_0001 $(TIME) gstlal_plot_psd_horizon $@ measured_psd.xml.gz @echo "" gstlal_bank.xml.gz: frames.cache # Produce template bank $(TIME) lalapps_tmpltbank \ --disable-compute-moments \ --grid-spacing Hexagonal \ --dynamic-range-exponent 69.0 \ --enable-high-pass 10.0 \ --high-pass-order 8 \ --strain-high-pass-order 8 \ --minimum-mass 1.35 \ --maximum-mass 1.45 \ --min-total-mass 2.7 \ --max-total-mass 2.9 \ --max-eta 0.25 \ --min-eta 0.18 \ --gps-start-time 1000000000 \ --gps-end-time 1000002048 \ --calibrated-data real_8 \ --channel-name H1:FAKE \ --space Tau0Tau3 \ --number-of-segments 15 \ --minimal-match 0.97 \ --high-pass-attenuation 0.1 \ --min-high-freq-cutoff ERD \ --segment-length 1048576 \ --low-frequency-cutoff 15.0 \ --num-freq-cutoffs 1 \ --sample-rate 4096 \ --high-frequency-cutoff 1024.0 \ --resample-filter ldas \ --strain-high-pass-atten 0.1 \ --strain-high-pass-freq 10.0 \ --frame-cache frames.cache \ --max-high-freq-cutoff ERD \ --approximant TaylorF2 \ --order twoPN \ --spectrum-type median \ --write-compress \ --verbose mv H1-TMPLTBANK-1000000000-2048.xml.gz $@ ligolw_no_ilwdchar $@ gstlal_inspiral_add_template_ids $@ @echo "" H1L1-ALL_MASS_MODEL-$(START)-$(DURATION).h5: gstlal_bank.xml.gz measured_psd.xml.gz # gstlal_bank_mass_model_0001 $(TIME) gstlal_inspiral_mass_model \ --template-bank gstlal_bank.xml.gz\ --reference-psd measured_psd.xml.gz \ --output H1L1-ALL_MASS_MODEL-$(START)-$(DURATION).h5 \ --model ligo @echo "" H1-0000_GSTLAL_SPLIT_BANK-0-0.xml.gz: gstlal_bank.xml.gz # gstlal_bank_splitter_0001 $(TIME) gstlal_bank_splitter \ --f-low $(LOW_FREQUENCY_CUTOFF) \ --group-by-chi $(NUM_CHI_BINS) \ --approximant $(APPROXIMANT) \ --output-cache H1_split_bank.cache \ --overlap $(OVERLAP) \ --instrument H1 \ --n $(NUM_SPLIT_TEMPLATES) \ --sort-by mchirp \ --max-f-final $(HIGH_FREQUENCY_CUTOFF) \ --num-banks $(NUMBANKS) \ $< @echo "" L1-0000_GSTLAL_SPLIT_BANK-0-0.xml.gz: gstlal_bank.xml.gz # gstlal_bank_splitter_0002 $(TIME) gstlal_bank_splitter \ --f-low $(LOW_FREQUENCY_CUTOFF) \ --group-by-chi $(NUM_CHI_BINS) \ --approximant $(APPROXIMANT) \ --output-cache L1_split_bank.cache \ --overlap $(OVERLAP) \ --instrument L1 \ --n $(NUM_SPLIT_TEMPLATES) \ --sort-by mchirp \ --max-f-final $(HIGH_FREQUENCY_CUTOFF) \ --num-banks $(NUMBANKS) \ $< @echo "" H1-0000_SVD-$(START)-$(DURATION).xml.gz: H1-0000_GSTLAL_SPLIT_BANK-0-0.xml.gz measured_psd.xml.gz # gstlal_svd_bank_0001 $(TIME) gstlal_svd_bank \ --reference-psd measured_psd.xml.gz \ --samples-min $(SAMPLES_MIN) \ --bank-id 0_0 \ --write-svd $@ \ --ortho-gate-fap 0.5 \ --template-bank $< \ --flow $(LOW_FREQUENCY_CUTOFF) \ --svd-tolerance 0.9999 \ --samples-max-64 2048 \ --clipleft 25 \ --autocorrelation-length 351 \ --samples-max-256 $(SAMPLES_MAX_256) \ --clipright 25 \ --samples-max $(SAMPLE_RATE) @echo "" L1-0000_SVD-$(START)-$(DURATION).xml.gz: L1-0000_GSTLAL_SPLIT_BANK-0-0.xml.gz measured_psd.xml.gz # gstlal_svd_bank_0002 $(TIME) gstlal_svd_bank \ --reference-psd measured_psd.xml.gz \ --samples-min $(SAMPLES_MIN) \ --bank-id 0_0 \ --write-svd $@ \ --ortho-gate-fap 0.5 \ --template-bank $< \ --flow $(LOW_FREQUENCY_CUTOFF) \ --svd-tolerance 0.9999 \ --samples-max-64 2048 \ --clipleft 25 \ --autocorrelation-length 351 \ --samples-max-256 $(SAMPLES_MAX_256) \ --clipright 25 \ --samples-max $(SAMPLE_RATE) @echo "" H1L1-0000_CREATE_PRIOR_DIST_STATS-$(START)-$(DURATION).xml.gz: H1L1-ALL_MASS_MODEL-$(START)-$(DURATION).h5 L1-0000_SVD-$(START)-$(DURATION).xml.gz H1-0000_SVD-$(START)-$(DURATION).xml.gz # gstlal_inspiral_create_prior_diststats_0001 $(TIME) gstlal_inspiral_create_prior_diststats \ --instrument H1 \ --instrument L1 \ --min-instruments $(MIN_IFOS) \ --write-likelihood H1L1-0000_CREATE_PRIOR_DIST_STATS-$(START)-$(DURATION).xml.gz \ --background-prior 1 \ --mass-model-file H1L1-ALL_MASS_MODEL-$(START)-$(DURATION).h5 \ --svd-file L1-0000_SVD-$(START)-$(DURATION).xml.gz \ --verbose @echo "" H1L1-0000_LLOID-$(START)-$(DURATION).xml.gz: measured_psd.xml.gz frames.cache segments.xml.gz vetoes.xml.gz tisi.xml H1-0000_SVD-$(START)-$(DURATION).xml.gz L1-0000_SVD-$(START)-$(DURATION).xml.gz H1L1-0000_CREATE_PRIOR_DIST_STATS-$(START)-$(DURATION).xml.gz # gstlal_inspiral_0001 $(TIME) gstlal_inspiral \ --reference-psd measured_psd.xml.gz \ --frame-segments-file segments.xml.gz \ --frame-cache frames.cache \ --veto-segments-file vetoes.xml.gz \ --time-slide-file tisi.xml \ --svd-bank H1:H1-0000_SVD-$(START)-$(DURATION).xml.gz,L1:L1-0000_SVD-$(START)-$(DURATION).xml.gz \ --output H1L1-0000_LLOID-$(START)-$(DURATION).xml.gz \ --ranking-stat-output H1L1-0000_DIST_STATS-$(START)-$(DURATION).xml.gz \ --gps-start-time $(START) \ --gps-end-time $(STOP) \ --data-source frames \ --frame-segments-name datasegments \ --channel-name H1=$(H1_CHANNEL) \ --channel-name L1=$(L1_CHANNEL) \ --track-psd \ --psd-fft-length 32 \ --fir-stride 1 \ --singles-threshold inf \ --tmp-space $(TMPDIR) \ --control-peak-time 0 \ --min-instruments $(MIN_IFOS) \ --ht-gate-threshold 100. \ --coincidence-threshold 0.005 \ --disable-service-discovery \ --verbose @echo "" H1L1-0000_DIST_STATS-$(START)-$(DURATION).xml.gz: H1L1-0000_LLOID-$(START)-$(DURATION).xml.gz H1L1-0000_MARG_DIST_STATS-$(START)-$(DURATION).xml.gz: H1L1-0000_CREATE_PRIOR_DIST_STATS-$(START)-$(DURATION).xml.gz H1L1-0000_DIST_STATS-$(START)-$(DURATION).xml.gz # gstlal_inspiral_marginalize_likelihood_0001 $(TIME) gstlal_inspiral_marginalize_likelihood \ --output $@ \ --marginalize ranking-stat \ $^ \ --verbose @echo "" H1L1-0000_CALC_RANK_PDFS-$(START)-$(DURATION).xml.gz: H1L1-0000_MARG_DIST_STATS-$(START)-$(DURATION).xml.gz # gstlal_inspiral_calc_rank_pdfs_0001 # n samples = 2**18 $(TIME) gstlal_inspiral_calc_rank_pdfs \ --output H1L1-0000_CALC_RANK_PDFS-$(START)-$(DURATION).xml.gz \ --ranking-stat-samples 262144 \ H1L1-0000_MARG_DIST_STATS-$(START)-$(DURATION).xml.gz \ --verbose @echo "" gstlal_inspiral_calc_likelihood_0001: H1L1-0000_LLOID-$(START)-$(DURATION).xml.gz H1L1-0000_MARG_DIST_STATS-$(START)-$(DURATION).xml.gz # gstlal_inspiral_calc_likelihood_0001 $(TIME) gstlal_inspiral_calc_likelihood \ --likelihood-url H1L1-0000_MARG_DIST_STATS-$(START)-$(DURATION).xml.gz \ --tmp-space $(TMPDIR) \ H1L1-0000_LLOID-$(START)-$(DURATION).xml.gz \ --verbose @touch gstlal_inspiral_calc_likelihood_0001 @echo "" lalapps_run_sqlite_0001: gstlal_inspiral_calc_likelihood_0001 # lalapps_run_sqlite_0001 $(TIME) lalapps_run_sqlite \ --sql-file $(LAL_PATH)/share/gstlal/simplify_and_cluster.sql \ --tmp-space $(TMPDIR) \ H1L1-0000_LLOID-$(START)-$(DURATION).xml.gz @touch lalapps_run_sqlite_0001 @echo "" H1L1-ALL_LLOID-$(START)-$(DURATION).sqlite: lalapps_run_sqlite_0001 # ligolw_sqlite_from_xml_0004 $(TIME) ligolw_sqlite \ --database H1L1-ALL_LLOID-$(START)-$(DURATION).sqlite \ --replace \ --tmp-space $(TMPDIR) \ vetoes.xml.gz \ segments.xml.gz \ H1L1-0000_LLOID-$(START)-$(DURATION).xml.gz \ --verbose @echo "" gstlal_inspiral_marginalize_likelihood_0002: H1L1-0000_CALC_RANK_PDFS-$(START)-$(DURATION).xml.gz # gstlal_inspiral_marginalize_likelihood_0002 $(TIME) gstlal_inspiral_marginalize_likelihood \ --output H1L1-0000_CALC_RANK_PDFS-$(START)-$(DURATION).xml.gz \ --marginalize ranking-stat-pdf \ H1L1-0000_CALC_RANK_PDFS-$(START)-$(DURATION).xml.gz \ --verbose @touch gstlal_inspiral_marginalize_likelihood_0002 @echo "" marginalized_likelihood.xml.gz: gstlal_inspiral_marginalize_likelihood_0002 # gstlal_inspiral_marginalize_likelihood_0003 $(TIME) gstlal_inspiral_marginalize_likelihood \ --output marginalized_likelihood.xml.gz \ --marginalize ranking-stat-pdf \ H1L1-0000_CALC_RANK_PDFS-$(START)-$(DURATION).xml.gz \ --verbose @echo "" gstlal_compute_far_from_snr_chisq_histograms_0001: marginalized_likelihood.xml.gz H1L1-ALL_LLOID-$(START)-$(DURATION).sqlite # gstlal_compute_far_from_snr_chisq_histograms_0001 $(TIME) gstlal_compute_far_from_snr_chisq_histograms \ --non-injection-db H1L1-ALL_LLOID-$(START)-$(DURATION).sqlite \ --background-bins-file marginalized_likelihood.xml.gz \ --tmp-space $(TMPDIR) \ --verbose @touch gstlal_compute_far_from_snr_chisq_histograms_0001 @echo "" #################### # PLotting scripts # #################### ifeq ($(ENABLE_PLOTTING), 1) gstlal_inspiral_plotsummary_0001: gstlal_compute_far_from_snr_chisq_histograms_0001 plots # gstlal_inspiral_plotsummary_0001 $(TIME) gstlal_inspiral_plotsummary \ --output-dir plots \ --remove-precession \ --user-tag ALL_LLOID_COMBINED \ --extend-veto-segments 8.0 \ --segments-name datasegments \ --likelihood-file post_marginalized_likelihood.xml.gz \ --shrink-data-segments 32.0 \ --tmp-space $(TMPDIR) \ H1L1-ALL_LLOID-$(START)-$(DURATION).sqlite \ --verbose @touch gstlal_inspiral_plotsummary_0001 @echo "" gstlal_inspiral_plot_background_0001: gstlal_inspiral_plotsummary_0001 # gstlal_inspiral_plot_background_0001 $(TIME) gstlal_inspiral_plot_background \ --database H1L1-ALL_LLOID-$(START)-$(DURATION).sqlite \ --user-tag ALL_LLOID_COMBINED \ --output-dir plots \ post_marginalized_likelihood.xml.gz @touch gstlal_inspiral_plot_background_0001 @echo "" ########################## # Generate summary pages # ########################## gstlal_inspiral_summary_page_0001: gstlal_inspiral_plot_background_0001 $(WEBDIR) # gstlal_inspiral_summary_page_0001 $(TIME) gstlal_inspiral_summary_page \ --output-user-tag ALL_LLOID_COMBINED \ --glob-path plots \ --webserver-dir $(WEBDIR) \ --title gstlal-$(START)-$(STOP)-closed-box @touch gstlal_inspiral_summary_page_0001 @echo "" gstlal_inspiral_summary_page_0002: gstlal_inspiral_summary_page_0001 # gstlal_inspiral_summary_page_0002 $(TIME) gstlal_inspiral_summary_page \ --output-user-tag ALL_LLOID_COMBINED \ --glob-path plots \ --webserver-dir $(WEBDIR)/OPEN-BOX \ --title gstlal-$(START)-$(STOP)-open-box \ --open-box @echo "Make box unreadable with: chmod a-rwx $(WEBDIR)/OPEN-BOX*" @echo "Open box with: chmod a+rwx $(WEBDIR)/OPEN-BOX*" @touch gstlal_inspiral_summary_page_0002 endif clean: rm -rf H* L* *.txt *.xml *.gz *.cache gstlal_* lalapps_* ligolw_* $(WEBDIR)* plots/ clean-data: rm -rf H-H1_FAKE-* L-L1_FAKE-* frames.cache clean-webdir: rm -rf $(WEBDIR)*