diff --git a/py/picca/bin/__init__.py b/py/picca/bin/__init__.py old mode 100644 new mode 100755 diff --git a/py/picca/bin/picca_Pk1D.py b/py/picca/bin/picca_Pk1D.py old mode 100644 new mode 100755 index 38db885c3..8a0df2215 --- a/py/picca/bin/picca_Pk1D.py +++ b/py/picca/bin/picca_Pk1D.py @@ -1,131 +1,164 @@ -#!/usr/bin/env python -"""Compute the 1D power spectrum -""" +#!/usr/bin/env python3 +"""Compute the 1D power spectrum""" import argparse import glob +import os +import sys from array import array -import numpy as np +from multiprocessing import Pool + import fitsio -import os +import numpy as np from picca import constants from picca.data import Delta -from picca.pk1d.compute_pk1d import (compute_correction_reso, - compute_correction_reso_matrix, compute_pk_noise, - compute_pk_raw, fill_masked_pixels, rebin_diff_noise, - split_forest, split_forest_in_z_parts, - check_linear_binning, Pk1D) +from picca.pk1d.compute_pk1d import ( + Pk1D, + check_linear_binning, + compute_correction_reso, + compute_correction_reso_matrix, + compute_pk_noise, + compute_pk_raw, + fill_masked_pixels, + rebin_diff_noise, + split_forest, + split_forest_in_z_parts, +) from picca.utils import userprint -from multiprocessing import Pool - # loop over input files -num_data=0 +num_data = 0 + + def process_all_files(index_file_args): global num_data file_index, file, args = index_file_args # If the healpix or tile number is given in the delta name, - # it names the Pk accordingly, else it takes + # it names the Pk accordingly, else it takes file_number = file.split("-")[-1].split(".fits.gz")[0] if not file_number.isdigit(): file_number = file_index if file_index % 5 == 0: - userprint("\rread {} of {} {}".format(file_index, args.len_files, - num_data), - end="") + userprint( + "\rread {} of {} {}".format(file_index, args.len_files, num_data), end="" + ) # read fits or ascii file - if args.in_format == 'fits': + if args.in_format == "fits": hdul = fitsio.FITS(file) try: deltas = [Delta.from_fitsio(hdu, pk1d_type=True) for hdu in hdul[1:]] running_on_raw_transmission = False except ValueError: - print("\nPk1d_type=True didn't work on read in, maybe perfect model? Trying without any noise or resolution corrections!") - deltas = [Delta.from_fitsio(hdu,pk1d_type=False) for hdu in hdul[1:]] + print( + "\nPk1d_type=True didn't work on read in, maybe perfect model? Trying without any noise or resolution corrections!" + ) + deltas = [Delta.from_fitsio(hdu, pk1d_type=False) for hdu in hdul[1:]] for delta in deltas: - delta.ivar=np.ones(delta.delta.shape)*1e10 - delta.mean_snr=1e5 - delta.mean_reso=1e-3 - delta.mean_reso_pix=1e-3 + delta.ivar = np.ones(delta.delta.shape) * 1e10 + delta.mean_snr = 1e5 + delta.mean_reso = 1e-3 + delta.mean_reso_pix = 1e-3 delta.exposures_diff = np.zeros(delta.delta.shape) running_on_raw_transmission = True - elif args.in_format == 'ascii': - ascii_file = open(file, 'r') + elif args.in_format == "ascii": + ascii_file = open(file, "r") deltas = [Delta.from_ascii(line) for line in ascii_file] running_on_raw_transmission = False - #add the check for linear binning on first spectrum only (assuming homogeneity within the file) + # add the check for linear binning on first spectrum only (assuming homogeneity within the file) delta = deltas[0] linear_binning, pixel_step = check_linear_binning(delta) if linear_binning: - #userprint("\n\nUsing linear binning, results will have units of AA") - if (args.disable_reso_matrix or not hasattr(delta, 'resolution_matrix') - or delta.resolution_matrix is None): + # userprint("\n\nUsing linear binning, results will have units of AA") + if ( + args.disable_reso_matrix + or not hasattr(delta, "resolution_matrix") + or delta.resolution_matrix is None + ): userprint( "Resolution matrix not found or disabled, using Gaussian resolution correction\n" ) reso_correction = "Gaussian" else: - #userprint("Using Resolution matrix for resolution correction\n") + # userprint("Using Resolution matrix for resolution correction\n") reso_correction = "matrix" else: - #userprint("\n\nUsing log binning, results will have units of km/s") + # userprint("\n\nUsing log binning, results will have units of km/s") reso_correction = "Gaussian" - #userprint("Using Gaussian resolution correction\n") + # userprint("Using Gaussian resolution correction\n") - use_exp_diff_cut=False - #add check if diff has ever been set + use_exp_diff_cut = False + # add check if diff has ever been set for delta in deltas: - #use this cut if some spectra have exposures_differences calculated - if not np.sum(delta.exposures_diff)==0: + # use this cut if some spectra have exposures_differences calculated + if not np.sum(delta.exposures_diff) == 0: use_exp_diff_cut = True break num_data += len(deltas) - #userprint("\n ndata = ", num_data) + # userprint("\n ndata = ", num_data) file_out = None for delta in deltas: - if (delta.mean_snr <= args.SNR_min - or delta.mean_reso >= args.reso_max): + if delta.mean_snr <= args.SNR_min or delta.mean_reso >= args.reso_max: continue if use_exp_diff_cut: - if np.sum(delta.exposures_diff)==0: + if np.sum(delta.exposures_diff) == 0: continue if linear_binning: - max_num_pixels_forest_theoretical=(1180-1050)*(delta.z_qso+1)/pixel_step #currently no min/max restframe values defined, so hard coding for the moment + max_num_pixels_forest_theoretical = ( + (1180 - 1050) * (delta.z_qso + 1) / pixel_step + ) # currently no min/max restframe values defined, so hard coding for the moment else: - max_num_pixels_forest_theoretical=(np.log(1180)-np.log10(1050))/pixel_step + max_num_pixels_forest_theoretical = ( + np.log(1180) - np.log10(1050) + ) / pixel_step # minimum number of pixel in forest if args.nb_pixel_min is not None: min_num_pixels = args.nb_pixel_min else: - min_num_pixels = int(args.nb_pixel_frac_min*max_num_pixels_forest_theoretical) #this is currently just hardcoding values so that spectra have a minimum length changing with z; but might be problematic for SBs - + min_num_pixels = int( + args.nb_pixel_frac_min * max_num_pixels_forest_theoretical + ) # this is currently just hardcoding values so that spectra have a minimum length changing with z; but might be problematic for SBs + if args.parts_in_redshift: # define chunks on a fixed redshift grid z_grid = np.array(args.z_parts) - split_array = split_forest_in_z_parts(z_grid, - delta.log_lambda, - delta.delta, - delta.exposures_diff, - delta.ivar, - min_num_pixels=min_num_pixels, - reso_matrix=(delta.resolution_matrix - if reso_correction == 'matrix' else None), - linear_binning=linear_binning) + split_array = split_forest_in_z_parts( + z_grid, + delta.log_lambda, + delta.delta, + delta.exposures_diff, + delta.ivar, + min_num_pixels=min_num_pixels, + reso_matrix=( + delta.resolution_matrix if reso_correction == "matrix" else None + ), + linear_binning=linear_binning, + ) num_parts = len(split_array[0]) - if reso_correction == 'matrix': - (mean_z_array, lambda_array, delta_array, exposures_diff_array, - ivar_array, reso_matrix_array) = split_array + if reso_correction == "matrix": + ( + mean_z_array, + lambda_array, + delta_array, + exposures_diff_array, + ivar_array, + reso_matrix_array, + ) = split_array else: - (mean_z_array, lambda_array, delta_array, exposures_diff_array, - ivar_array) = split_array + ( + mean_z_array, + lambda_array, + delta_array, + exposures_diff_array, + ivar_array, + ) = split_array if not linear_binning: log_lambda_array = lambda_array @@ -134,10 +167,13 @@ def process_all_files(index_file_args): # first pixel in forest selected_pixels = 10**delta.log_lambda > args.lambda_obs_min - #this works as selected_pixels returns a bool and argmax points - #towards the first occurance for equal values - first_pixel_index = (np.argmax(selected_pixels) if - np.any(selected_pixels) else len(selected_pixels)) + # this works as selected_pixels returns a bool and argmax points + # towards the first occurance for equal values + first_pixel_index = ( + np.argmax(selected_pixels) + if np.any(selected_pixels) + else len(selected_pixels) + ) # minimum number of pixel in forest min_num_pixels = args.nb_pixel_min @@ -145,11 +181,12 @@ def process_all_files(index_file_args): continue # Split the forest in n parts - max_num_parts = (len(delta.log_lambda) - - first_pixel_index) // min_num_pixels + max_num_parts = ( + len(delta.log_lambda) - first_pixel_index + ) // min_num_pixels num_parts = min(args.nb_part, max_num_parts) - #the split_forest function works with either binning, but needs to be uniform + # the split_forest function works with either binning, but needs to be uniform if linear_binning: split_array = split_forest( num_parts, @@ -159,69 +196,126 @@ def process_all_files(index_file_args): delta.exposures_diff, delta.ivar, first_pixel_index, - reso_matrix=(delta.resolution_matrix - if reso_correction == 'matrix' else None), - linear_binning=True) - if reso_correction == 'matrix': - (mean_z_array, lambda_array, delta_array, exposures_diff_array, - ivar_array, reso_matrix_array) = split_array + reso_matrix=( + delta.resolution_matrix if reso_correction == "matrix" else None + ), + linear_binning=True, + ) + if reso_correction == "matrix": + ( + mean_z_array, + lambda_array, + delta_array, + exposures_diff_array, + ivar_array, + reso_matrix_array, + ) = split_array else: - (mean_z_array, lambda_array, delta_array, exposures_diff_array, - ivar_array) = split_array + ( + mean_z_array, + lambda_array, + delta_array, + exposures_diff_array, + ivar_array, + ) = split_array else: - (mean_z_array, log_lambda_array, delta_array, exposures_diff_array, - ivar_array) = split_forest(num_parts, pixel_step, - delta.log_lambda, delta.delta, - delta.exposures_diff, delta.ivar, - first_pixel_index) + ( + mean_z_array, + log_lambda_array, + delta_array, + exposures_diff_array, + ivar_array, + ) = split_forest( + num_parts, + pixel_step, + delta.log_lambda, + delta.delta, + delta.exposures_diff, + delta.ivar, + first_pixel_index, + ) - #the rebin_diff_noise function works with either binning, but needs to be uniform + # the rebin_diff_noise function works with either binning, but needs to be uniform for part_index in range(num_parts): # rebin exposures_diff spectrum - if (args.noise_estimate == 'rebin_diff' - or args.noise_estimate == 'mean_rebin_diff'): + if ( + args.noise_estimate == "rebin_diff" + or args.noise_estimate == "mean_rebin_diff" + ): if linear_binning: exposures_diff_array[part_index] = rebin_diff_noise( - pixel_step, lambda_array[part_index], - exposures_diff_array[part_index]) + pixel_step, + lambda_array[part_index], + exposures_diff_array[part_index], + ) else: exposures_diff_array[part_index] = rebin_diff_noise( - pixel_step, log_lambda_array[part_index], - exposures_diff_array[part_index]) + pixel_step, + log_lambda_array[part_index], + exposures_diff_array[part_index], + ) # Fill masked pixels with 0. - #the fill_masked_pixels function works with either binning, but needs to be uniform + # the fill_masked_pixels function works with either binning, but needs to be uniform if linear_binning: - #the resolution matrix does not need to have pixels filled in any way... - (lambda_new, delta_new, exposures_diff_new, ivar_new, - num_masked_pixels) = fill_masked_pixels( - pixel_step, lambda_array[part_index], - delta_array[part_index], exposures_diff_array[part_index], - ivar_array[part_index], args.no_apply_filling) + # the resolution matrix does not need to have pixels filled in any way... + ( + lambda_new, + delta_new, + exposures_diff_new, + ivar_new, + num_masked_pixels, + ) = fill_masked_pixels( + pixel_step, + lambda_array[part_index], + delta_array[part_index], + exposures_diff_array[part_index], + ivar_array[part_index], + args.no_apply_filling, + ) else: - (log_lambda_new, delta_new, exposures_diff_new, ivar_new, - num_masked_pixels) = fill_masked_pixels( - pixel_step, log_lambda_array[part_index], - delta_array[part_index], exposures_diff_array[part_index], - ivar_array[part_index], args.no_apply_filling) - - if (args.nb_pixel_masked_max is not None): + ( + log_lambda_new, + delta_new, + exposures_diff_new, + ivar_new, + num_masked_pixels, + ) = fill_masked_pixels( + pixel_step, + log_lambda_array[part_index], + delta_array[part_index], + exposures_diff_array[part_index], + ivar_array[part_index], + args.no_apply_filling, + ) + + if args.nb_pixel_masked_max is not None: max_num_masked_pixels = args.nb_pixel_masked_max elif linear_binning: - max_num_masked_pixels = int(args.nb_pixel_masked_frac_max*(np.max(lambda_new)-np.min(lambda_new))/pixel_step) #this only accounts for masking inside the spectrum, not at the adges + max_num_masked_pixels = int( + args.nb_pixel_masked_frac_max + * (np.max(lambda_new) - np.min(lambda_new)) + / pixel_step + ) # this only accounts for masking inside the spectrum, not at the adges else: - max_num_masked_pixels = int(args.nb_pixel_masked_frac_max*(np.max(log_lambda_new)-np.min(log_lambda_new))/pixel_step) - + max_num_masked_pixels = int( + args.nb_pixel_masked_frac_max + * (np.max(log_lambda_new) - np.min(log_lambda_new)) + / pixel_step + ) + if num_masked_pixels > max_num_masked_pixels: continue # Compute pk_raw, needs uniform binning - k, fft_delta, pk_raw = compute_pk_raw(pixel_step, - delta_new, - linear_binning=linear_binning) + k, fft_delta, pk_raw = compute_pk_raw( + pixel_step, delta_new, linear_binning=linear_binning + ) # Compute pk_noise - if (args.noise_estimate == 'pipeline')|(args.noise_estimate == 'mean_pipeline'): + if (args.noise_estimate == "pipeline") | ( + args.noise_estimate == "mean_pipeline" + ): run_gaussian_noise = True else: run_gaussian_noise = False @@ -235,70 +329,87 @@ def process_all_files(index_file_args): num_noise_exposures=args.num_noise_exp, ) else: - pk_noise=pk_diff=np.zeros(pk_raw.shape) - fft_delta_noise = np.zeros(pk_raw.shape,dtype=np.complex128) - fft_delta_diff = np.zeros(pk_raw.shape,dtype=np.complex128) + pk_noise = pk_diff = np.zeros(pk_raw.shape) + fft_delta_noise = np.zeros(pk_raw.shape, dtype=np.complex128) + fft_delta_diff = np.zeros(pk_raw.shape, dtype=np.complex128) # Compute resolution correction, needs uniform binning if not running_on_raw_transmission: if linear_binning: - #in this case all is in AA space - if reso_correction == 'matrix': + # in this case all is in AA space + if reso_correction == "matrix": correction_reso = compute_correction_reso_matrix( - reso_matrix=np.mean(reso_matrix_array[part_index], - axis=1), + reso_matrix=np.mean(reso_matrix_array[part_index], axis=1), k=k, delta_pixel=pixel_step, num_pixel=len(lambda_new), - pixelization_correction = args.add_pixelization_correction) - elif reso_correction == 'Gaussian': - #this is roughly converting the mean resolution estimate back to pixels - #and then multiplying with pixel size + pixelization_correction=args.add_pixelization_correction, + ) + elif reso_correction == "Gaussian": + # this is roughly converting the mean resolution estimate back to pixels + # and then multiplying with pixel size mean_reso_AA = pixel_step * delta.mean_reso_pix correction_reso = compute_correction_reso( - delta_pixel=pixel_step, mean_reso=mean_reso_AA, k=k) + delta_pixel=pixel_step, mean_reso=mean_reso_AA, k=k + ) else: - #in this case all is in velocity space - delta_pixel = (pixel_step * np.log(10.) * - constants.speed_light / 1000.) + # in this case all is in velocity space + delta_pixel = ( + pixel_step * np.log(10.0) * constants.speed_light / 1000.0 + ) correction_reso = compute_correction_reso( - delta_pixel=delta_pixel, mean_reso=delta.mean_reso, k=k) + delta_pixel=delta_pixel, mean_reso=delta.mean_reso, k=k + ) else: - correction_reso= compute_correction_reso(delta_pixel=pixel_step, mean_reso=0., k=k) + correction_reso = compute_correction_reso( + delta_pixel=pixel_step, mean_reso=0.0, k=k + ) # Compute 1D Pk - if args.noise_estimate == 'pipeline' or running_on_raw_transmission: + if args.noise_estimate == "pipeline" or running_on_raw_transmission: pk = (pk_raw - pk_noise) / correction_reso - elif args.noise_estimate == 'mean_pipeline': + elif args.noise_estimate == "mean_pipeline": if args.kmin_noise_avg is None and linear_binning: - #this is roughly the same range as eBOSS analyses for z=2.2 + # this is roughly the same range as eBOSS analyses for z=2.2 selection = (k > 0) & (k < 1.5) elif args.kmin_noise_avg is None: selection = (k > 0) & (k < 0.02) else: - selection = (((k > args.kmin_noise_avg) if args.kmax_noise_avg is not None else 1) & - ((k < args.kmax_noise_avg) if args.kmax_noise_avg is not None else 1)) + selection = ( + (k > args.kmin_noise_avg) + if args.kmax_noise_avg is not None + else 1 + ) & ( + (k < args.kmax_noise_avg) + if args.kmax_noise_avg is not None + else 1 + ) mean_pk_noise = np.mean(pk_noise[selection]) pk = (pk_raw - mean_pk_noise) / correction_reso - - - elif (args.noise_estimate == 'diff' or args.noise_estimate == 'rebin_diff'): + elif args.noise_estimate == "diff" or args.noise_estimate == "rebin_diff": pk = (pk_raw - pk_diff) / correction_reso - elif (args.noise_estimate == 'mean_diff' or 'mean_rebin_diff'): + elif args.noise_estimate == "mean_diff" or "mean_rebin_diff": if args.kmin_noise_avg is None and linear_binning: - #this is roughly the same range as eBOSS analyses for z=2.2 + # this is roughly the same range as eBOSS analyses for z=2.2 selection = (k > 0) & (k < 1.5) elif args.kmin_noise_avg is None: selection = (k > 0) & (k < 0.02) else: - selection = (((k > args.kmin_noise_avg) if args.kmax_noise_avg is not None else 1) & - ((k < args.kmax_noise_avg) if args.kmax_noise_avg is not None else 1)) + selection = ( + (k > args.kmin_noise_avg) + if args.kmax_noise_avg is not None + else 1 + ) & ( + (k < args.kmax_noise_avg) + if args.kmax_noise_avg is not None + else 1 + ) mean_pk_diff = np.mean(pk_diff[selection]) pk = (pk_raw - mean_pk_diff) / correction_reso # save in fits format - if args.out_format == 'fits': + if args.out_format == "fits": pk1d_class = Pk1D( ra=delta.ra, dec=delta.dec, @@ -317,19 +428,20 @@ def process_all_files(index_file_args): correction_reso=correction_reso, pk=pk, fft_delta=fft_delta, - fft_delta_noise = fft_delta_noise, - fft_delta_diff = fft_delta_diff, + fft_delta_noise=fft_delta_noise, + fft_delta_diff=fft_delta_diff, ) - + if file_out is None: - file_out = fitsio.FITS((args.out_dir + '/Pk1D-' + - str(file_number) + '.fits.gz'), - 'rw', - clobber=True) - + file_out = fitsio.FITS( + (args.out_dir + "/Pk1D-" + str(file_number) + ".fits.gz"), + "rw", + clobber=True, + ) + pk1d_class.write_fits(file_out) - if (args.out_format == 'fits' and file_out is not None): + if args.out_format == "fits" and file_out is not None: file_out.close() return 0 @@ -341,216 +453,265 @@ def main(cmdargs=None): Uses the resolution matrix correction for DESI data""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Compute the 1D power spectrum') - - parser.add_argument('--out-dir', - type=str, - default=None, - required=True, - help='Output directory') - - parser.add_argument('--out-format', - type=str, - default='fits', - required=False, - help='Output format: ascii or fits') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') + description="Compute the 1D power spectrum", + ) parser.add_argument( - '--in-format', + "--out-dir", type=str, default=None, required=True, help="Output directory" + ) + + parser.add_argument( + "--out-format", + type=str, + default="fits", + required=False, + help="Output format: ascii or fits", + ) + + parser.add_argument( + "--in-dir", type=str, - default='fits', + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--in-format", + type=str, + default="fits", required=False, - help=' Input format used for input files: ascii or fits') - - parser.add_argument('--SNR-min', - type=float, - default=2., - required=False, - help='Minimal mean SNR per pixel ') - - parser.add_argument('--reso-max', - type=float, - default=85., - required=False, - help='Maximal resolution in km/s ') - - parser.add_argument('--lambda-obs-min', - type=float, - default=3600., - required=False, - help='Lower limit on observed wavelength [Angstrom]') - - parser.add_argument('--nb-part', - type=int, - default=3, - required=False, - help='Number of parts (chunks) in forest') - - parser.add_argument('--parts-in-redshift', - action='store_true', - default=False, - required=False, - help='Define forest parts (chunks) as a function of redshift (observer frame) instead of rest frame') - - parser.add_argument('--z-parts', - nargs='+', type=float, - default=[2.1, 2.3, 2.5, 2.7, 2.9, 3.1, 3.3, 3.5, 3.7, 3.9, 4.1], - required=False, - help='Redshift bin edges for forest parts (chunks) when parts-in-redshift is set') - - parser.add_argument('--nb-pixel-min', - type=int, - default=None, - required=False, - help='Minimal number of pixels in a part of forest') - - - parser.add_argument('--nb-pixel-frac-min', - type=float, - default=None, - required=False, - help='Minimal number of pixels in a part of forest (default is assuming ~577 pixels for a z=2.5 QSO in the forest of 1050-1180A and masking up to 75)') + help=" Input format used for input files: ascii or fits", + ) parser.add_argument( - '--nb-pixel-masked-max', + "--SNR-min", + type=float, + default=2.0, + required=False, + help="Minimal mean SNR per pixel ", + ) + + parser.add_argument( + "--reso-max", + type=float, + default=85.0, + required=False, + help="Maximal resolution in km/s ", + ) + + parser.add_argument( + "--lambda-obs-min", + type=float, + default=3600.0, + required=False, + help="Lower limit on observed wavelength [Angstrom]", + ) + + parser.add_argument( + "--nb-part", + type=int, + default=3, + required=False, + help="Number of parts (chunks) in forest", + ) + + parser.add_argument( + "--parts-in-redshift", + action="store_true", + default=False, + required=False, + help="Define forest parts (chunks) as a function of redshift (observer frame) instead of rest frame", + ) + + parser.add_argument( + "--z-parts", + nargs="+", + type=float, + default=[2.1, 2.3, 2.5, 2.7, 2.9, 3.1, 3.3, 3.5, 3.7, 3.9, 4.1], + required=False, + help="Redshift bin edges for forest parts (chunks) when parts-in-redshift is set", + ) + + parser.add_argument( + "--nb-pixel-min", + type=int, + default=None, + required=False, + help="Minimal number of pixels in a part of forest", + ) + + parser.add_argument( + "--nb-pixel-frac-min", + type=float, + default=None, + required=False, + help="Minimal number of pixels in a part of forest (default is assuming ~577 pixels for a z=2.5 QSO in the forest of 1050-1180A and masking up to 75)", + ) + + parser.add_argument( + "--nb-pixel-masked-max", type=int, - default=None,#40, + default=None, # 40, required=False, - help='Maximal number of masked pixels in a part of forest') - + help="Maximal number of masked pixels in a part of forest", + ) + parser.add_argument( - '--nb-pixel-masked-frac-max', + "--nb-pixel-masked-frac-max", type=float, default=None, required=False, - help='Maximal number of masked pixels in a part of forest (default is 21%% of the forest length, i.e. similar to the previous value at z=2.5 for a 3 chunk spectrum and 1050-1180A)') + help="Maximal number of masked pixels in a part of forest (default is 21%% of the forest length, i.e. similar to the previous value at z=2.5 for a 3 chunk spectrum and 1050-1180A)", + ) - parser.add_argument('--no-apply-filling', - action='store_true', - default=False, - required=False, - help='Dont fill masked pixels') + parser.add_argument( + "--no-apply-filling", + action="store_true", + default=False, + required=False, + help="Dont fill masked pixels", + ) parser.add_argument( - '--noise-estimate', + "--noise-estimate", type=str, - default='mean_diff', + default="mean_diff", required=False, - help=('Estimate of Pk_noise ' - 'pipeline/mean_pipeline/diff/mean_diff/rebin_diff/mean_rebin_diff')) + help=( + "Estimate of Pk_noise " + "pipeline/mean_pipeline/diff/mean_diff/rebin_diff/mean_rebin_diff" + ), + ) - parser.add_argument('--forest-type', - type=str, - default='Lya', - required=False, - help='Forest used: Lya, SiIV, CIV') + parser.add_argument( + "--forest-type", + type=str, + default="Lya", + required=False, + help="Forest used: Lya, SiIV, CIV", + ) parser.add_argument( - '--abs-igm', + "--abs-igm", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption line in picca.constants defining the ' - 'redshift of the forest pixels')) + help=( + "Name of the absorption line in picca.constants defining the " + "redshift of the forest pixels" + ), + ) - #additional options - parser.add_argument('--num-processors', - type=int, - default=1, - required=False, - help='Number of processors to use for computation') + # additional options + parser.add_argument( + "--num-processors", + type=int, + default=1, + required=False, + help="Number of processors to use for computation", + ) parser.add_argument( - '--num-noise-exp', + "--num-noise-exp", default=100, type=int, required=False, - help='number of pipeline noise realizations to generate per spectrum') - - parser.add_argument('--disable-reso-matrix', - default=False, - action='store_true', - required=False, - help=('do not use the resolution matrix even ' - 'if it exists and we are on linear binning')) - - parser.add_argument('--add-pixelization-correction', - default=False, - action='store_true', - required=False, - help=('Add a pixelization correction, as if the resolution ' - 'matrix was doubly pixelized. Only use this option in ' - 'quickquasars mocks')) + help="number of pipeline noise realizations to generate per spectrum", + ) + + parser.add_argument( + "--disable-reso-matrix", + default=False, + action="store_true", + required=False, + help=( + "do not use the resolution matrix even " + "if it exists and we are on linear binning" + ), + ) + + parser.add_argument( + "--add-pixelization-correction", + default=False, + action="store_true", + required=False, + help=( + "Add a pixelization correction, as if the resolution " + "matrix was doubly pixelized. Only use this option in " + "quickquasars mocks" + ), + ) parser.add_argument( - '--seed', + "--seed", default=4, required=False, type=int, - help= - ('seed for random number generator, default 4, let system determine seed if set to 0' - )) - + help=( + "seed for random number generator, default 4, let system determine seed if set to 0" + ), + ) parser.add_argument( - '--kmin_noise_avg', + "--kmin_noise_avg", default=None, required=False, type=float, - help= - ('minimal mode to take into account when computing noise/diff power average' - )) + help=( + "minimal mode to take into account when computing noise/diff power average" + ), + ) parser.add_argument( - '--kmax_noise_avg', + "--kmax_noise_avg", default=None, required=False, type=float, - help= - ('maximal mode to take into account when computing noise/diff power average' - )) + help=( + "maximal mode to take into account when computing noise/diff power average" + ), + ) args = parser.parse_args(cmdargs) - if args.seed==0: - seed=None + if args.seed == 0: + seed = None else: - seed=args.seed + seed = args.seed # Read deltas - if args.in_format == 'fits': - files = sorted(glob.glob(args.in_dir + "/*.fits.gz") + - glob.glob(args.in_dir + "/*.fits")) - elif args.in_format == 'ascii': + if args.in_format == "fits": + files = sorted( + glob.glob(args.in_dir + "/*.fits.gz") + glob.glob(args.in_dir + "/*.fits") + ) + elif args.in_format == "ascii": files = sorted(glob.glob(args.in_dir + "/*.txt")) # initialize randoms np.random.seed(seed) userprint(f"Computing Pk1d for {args.in_dir} ({len(files)} files to process)") args.len_files = len(files) - #create output dir if it does not exist + # create output dir if it does not exist os.makedirs(args.out_dir, exist_ok=True) if args.nb_pixel_min is None and args.nb_pixel_frac_min is None: - # could make this fraction a new default, but that would probably cause trouble for SBs - # args.nb_pixel_frac_min = 0.13 #this is a suggestion for a new default - args.nb_pixel_min = 75 #this is the previous default + # could make this fraction a new default, but that would probably cause trouble for SBs + # args.nb_pixel_frac_min = 0.13 #this is a suggestion for a new default + args.nb_pixel_min = 75 # this is the previous default elif not (args.nb_pixel_frac_min is None or args.nb_pixel_min is None): print("both nb_pixel_frac_min and nb_pixel_min were set, using the latter") - args.nb_pixel_frac_min=None + args.nb_pixel_frac_min = None if args.nb_pixel_masked_frac_max is None and args.nb_pixel_masked_max is None: - # could make this, i.e. 10% of the estimated forest length the new default - # args.nb_pixel_masked_frac_max = 0.21 #this is a suggestion for a new default - args.nb_pixel_masked_max = 40 #this is the previous default - elif not (args.nb_pixel_masked_frac_max is None or args.nb_pixel_masked_max is None): - print("both nb_pixel_masked_frac_max and nb_pixel_masked_max were set, using the latter") - args.nb_pixel_masked_frac_max=None - + # could make this, i.e. 10% of the estimated forest length the new default + # args.nb_pixel_masked_frac_max = 0.21 #this is a suggestion for a new default + args.nb_pixel_masked_max = 40 # this is the previous default + elif not ( + args.nb_pixel_masked_frac_max is None or args.nb_pixel_masked_max is None + ): + print( + "both nb_pixel_masked_frac_max and nb_pixel_masked_max were set, using the latter" + ) + args.nb_pixel_masked_frac_max = None print([[i, f] for i, f in enumerate(files)]) @@ -561,3 +722,8 @@ def main(cmdargs=None): else: [process_all_files((i, f, args)) for i, f in enumerate(files)] userprint("all done ") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_Pk1D_average_mean.py b/py/picca/bin/picca_Pk1D_average_mean.py old mode 100644 new mode 100755 index 7bf55d233..2bb7f19de --- a/py/picca/bin/picca_Pk1D_average_mean.py +++ b/py/picca/bin/picca_Pk1D_average_mean.py @@ -1,8 +1,11 @@ -#!/usr/bin/env python -"""Compute the averaged 1D power spectrum -""" +#!/usr/bin/env python3 +"""Compute the averaged 1D power spectrum""" + +import argparse +import ast +import glob +import sys -import sys, glob, argparse, ast from picca.pk1d import postproc_pk1d from picca.utils import userprint @@ -69,3 +72,8 @@ def main(cmdargs=None): args.output_path, weighted_mean=False, ) + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_Pk1D_cross_exposure.py b/py/picca/bin/picca_Pk1D_cross_exposure.py old mode 100644 new mode 100755 index 98bc18c1d..460e1c199 --- a/py/picca/bin/picca_Pk1D_cross_exposure.py +++ b/py/picca/bin/picca_Pk1D_cross_exposure.py @@ -1,14 +1,18 @@ -#!/usr/bin/env python -"""Compute the individual cross-exposure 1D power spectra -""" +#!/usr/bin/env python3 +"""Compute the individual cross-exposure 1D power spectra""" + +import argparse +import glob +import multiprocessing as mp +import os +import sys +from functools import partial -import os, argparse, glob import fitsio import numpy as np -from picca.pk1d.compute_pk1d import compute_pk_cross_exposure, Pk1D + +from picca.pk1d.compute_pk1d import Pk1D, compute_pk_cross_exposure from picca.utils import userprint -import multiprocessing as mp -from functools import partial def treat_pk_file(out_dir, filename): @@ -16,9 +20,9 @@ def treat_pk_file(out_dir, filename): Takes a single file containing the FFT of delta for multiple exposures and computes the cross-exposure power spectrum. The function returns nothing, but writes to disk a new fits file with all the information needed - to compute Pk_cross_exposure. This is done by looping over each targetid and chunkid, + to compute Pk_cross_exposure. This is done by looping over each targetid and chunkid, and computing Pk_cross_exposure for each pair of exposures. - + Arguments --------- out_dir: string @@ -196,3 +200,8 @@ def main(cmdargs=None): else: with mp.Pool(args.num_processors) as pool: pool.map(func, files) + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_Pk1D_postprocess.py b/py/picca/bin/picca_Pk1D_postprocess.py old mode 100644 new mode 100755 index 5358a6d4d..891349189 --- a/py/picca/bin/picca_Pk1D_postprocess.py +++ b/py/picca/bin/picca_Pk1D_postprocess.py @@ -1,258 +1,350 @@ -#!/usr/bin/env python -"""Compute the averaged 1D power spectrum -""" +#!/usr/bin/env python3 +"""Compute the averaged 1D power spectrum""" + +import argparse +import os +import sys -import os, argparse -import numpy as np import fitsio +import numpy as np + from picca.pk1d import postproc_pk1d, utils def define_wavenumber_array(k_min, k_max, k_dist, velunits, pixsize, rebinfac): - """ Define the wavenumber array limits and binning - Default binning defined linearly with assumed average redshift of 3.4, - and a forest defined between 1050 and 1200 Angstrom. - Default velocity binning with same number of pixels""" + """Define the wavenumber array limits and binning + Default binning defined linearly with assumed average redshift of 3.4, + and a forest defined between 1050 and 1200 Angstrom. + Default velocity binning with same number of pixels""" k_min_lin_default = utils.DEFAULT_K_MIN_LIN * rebinfac k_max_lin_default = np.pi / pixsize - nb_k_bins = int(k_max_lin_default / k_min_lin_default / utils.DEFAULT_K_BINNING_FACTOR) + nb_k_bins = int( + k_max_lin_default / k_min_lin_default / utils.DEFAULT_K_BINNING_FACTOR + ) k_dist_lin_default = (k_max_lin_default - k_min_lin_default) / nb_k_bins k_dist_vel_default = utils.DEFAULT_K_BIN_VEL * rebinfac k_max_vel_default = utils.DEFAULT_K_MIN_VEL + nb_k_bins * k_dist_vel_default if velunits: - if k_min is None: k_min = utils.DEFAULT_K_MIN_VEL - if k_max is None: k_max = k_max_vel_default - if k_dist is None: k_dist = k_dist_vel_default + if k_min is None: + k_min = utils.DEFAULT_K_MIN_VEL + if k_max is None: + k_max = k_max_vel_default + if k_dist is None: + k_dist = k_dist_vel_default else: - if k_min is None: k_min = k_min_lin_default - if k_max is None: k_max = k_max_lin_default - if k_dist is None: k_dist = k_dist_lin_default + if k_min is None: + k_min = k_min_lin_default + if k_max is None: + k_max = k_max_lin_default + if k_dist is None: + k_dist = k_dist_lin_default return k_min, k_max, k_dist - def main(cmdargs=None): """Compute the averaged 1D power spectrum""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Compute the averaged 1D power spectrum') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to individual P1D files') - - parser.add_argument('--output-file', - type=str, - default=None, - required=False, - help='Output file name,' - 'If set to None, file name is set to --in-dir/mean_Pk1d_[weight_method]_[snr_cut]_[vel].fits.gz') - - parser.add_argument('--zedge-min', - type=float, - default='2.1', - required=False, - help='Minimal value of the redshift edge array,' - 'Default value: 2.1') - - parser.add_argument('--zedge-max', - type=float, - default='6.5', - required=False, - help='Maximal value of the redshift edge array,' - 'Default value: 6.5') - - parser.add_argument('--zedge-bin', - type=float, - default='0.2', - required=False, - help='Number of bins of the redshift edge array,' - 'Default value: 0.2') - - parser.add_argument('--kedge-min', - type=float, - default=None, - required=False, - help='Minimal value of the wavenumber edges array,' - 'Default value defined as function of --rebinfac, ' - '--pixsize, and --velunits arguments') - - parser.add_argument('--kedge-max', - type=float, - default=None, - required=False, - help='Maximal value of the wavenumber edges array,' - 'Default value defined as function of --rebinfac, ' - '--pixsize, and --velunits arguments') - - parser.add_argument('--kedge-bin', - type=float, - default=None, - required=False, - help='Number of bins of the wavenumber edges array,' - 'Default value defined as function of --rebinfac, ' - '--pixsize, and --velunits arguments') - - parser.add_argument('--rebinfac', - type=int, - default=1, - required=False, - help='Rebinning factor used to define the binning of ' - 'the output wavenumber array') - - parser.add_argument('--pixsize', - type=float, - default=0.8, - required=False, - help='Size of a spectrum pixel in Angstrom, used to' - 'define the binning of the output wavenumber array') - - parser.add_argument('--weight-method', - type=str, - default='no_weights', - help='SNR weighting scheme for the mean P1D computation,' - 'Possible options: no_weights, fit_snr') - - parser.add_argument('--apply_z_weights', - action='store_true', - default=False, - required=False, - help='If set, apply a z weighting scheme analog to that used for QMLE (eg. 2008.06421). ' - 'Each chunk with mean redshift z contributes two nearest bins z_i (resp. z_j=z_i+dz), ' - 'with weights |z-z_j|/dz (resp. |z-z_i|/dz).') - - parser.add_argument('--output-snrfit', - type=str, - default=None, - help='Name of the ASCII file where SNR fit results are stored,' - 'if weight-method is fit_snr') - - parser.add_argument('--snr-cut-scheme', - type=str, - default=None, - help='Choice of SNR cut type, ' - 'Possible options: eboss (varying cut vs z, as in eBOSS - DR14); fixed;' - 'None (obligatory when --weight-method != no_weights)') - - parser.add_argument('--snrcut', - type=float, - default=None, - help='Value of the SNR cut if snr-cut-scheme=fixed') - - parser.add_argument('--skymask-file', - type=str, - default=None, - help='Name of the ASCII file containing skyline masks, which was used to' - 'compute the deltas. If set, a correction for this mask will be computed,' - 'with "*_skycorr" columns in the output.') - - parser.add_argument('--minwave-skymask', - type=float, - default=utils.DEFAULT_MINWAVE_SKYMASK, - help='Parameter for the wavelength grid used for the skymask,' - 'if --skymask-file is set.') - - parser.add_argument('--maxwave-skymask', - type=float, - default=utils.DEFAULT_MAXWAVE_SKYMASK, - help='Parameter for the wavelength grid used for the skymask,' - 'if --skymask-file is set.') - - parser.add_argument('--dwave-skymask', - type=float, - default=utils.DEFAULT_DWAVE_SKYMASK, - help='Parameter for the wavelength grid used for the skymask,' - 'if --skymask-file is set.') - - parser.add_argument('--overwrite', - action='store_true', - default=False, - required=False, - help='Overwrite the output') - - parser.add_argument('--velunits', - action='store_true', - default=False, - required=False, - help='Compute mean P1D in velocity units') - - parser.add_argument('--no-median', - action='store_true', - default=False, - required=False, - help='Skip computation of median quantities') - - parser.add_argument('--ncpu', - type=int, - default=8, - required=False, - help='Number of CPUs used to read input P1D files') - - parser.add_argument('--covariance', - action='store_true', - default=False, - required=False, - help='Compute covariance matrix') - - parser.add_argument('--bootstrap', - action='store_true', - default=False, - required=False, - help='Compute covariance matrix with bootstrap method') - - parser.add_argument('--nbootstrap', - type=int, - default=20, - required=False, - help='Number of bootstrap iteration used') - - parser.add_argument('--bootstrap_average', - action='store_true', - default=False, - required=False, - help='Compute covariance matrix with bootstrap method on the average P1D') - - parser.add_argument('--nbootstrap_average', - type=int, - default=1000, - required=False, - help='Number of bootstrap iteration used for average bootstrap') - + description="Compute the averaged 1D power spectrum", + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to individual P1D files", + ) + + parser.add_argument( + "--output-file", + type=str, + default=None, + required=False, + help="Output file name," + "If set to None, file name is set to --in-dir/mean_Pk1d_[weight_method]_[snr_cut]_[vel].fits.gz", + ) + + parser.add_argument( + "--zedge-min", + type=float, + default="2.1", + required=False, + help="Minimal value of the redshift edge array," "Default value: 2.1", + ) + + parser.add_argument( + "--zedge-max", + type=float, + default="6.5", + required=False, + help="Maximal value of the redshift edge array," "Default value: 6.5", + ) + + parser.add_argument( + "--zedge-bin", + type=float, + default="0.2", + required=False, + help="Number of bins of the redshift edge array," "Default value: 0.2", + ) + + parser.add_argument( + "--kedge-min", + type=float, + default=None, + required=False, + help="Minimal value of the wavenumber edges array," + "Default value defined as function of --rebinfac, " + "--pixsize, and --velunits arguments", + ) + + parser.add_argument( + "--kedge-max", + type=float, + default=None, + required=False, + help="Maximal value of the wavenumber edges array," + "Default value defined as function of --rebinfac, " + "--pixsize, and --velunits arguments", + ) + + parser.add_argument( + "--kedge-bin", + type=float, + default=None, + required=False, + help="Number of bins of the wavenumber edges array," + "Default value defined as function of --rebinfac, " + "--pixsize, and --velunits arguments", + ) + + parser.add_argument( + "--rebinfac", + type=int, + default=1, + required=False, + help="Rebinning factor used to define the binning of " + "the output wavenumber array", + ) + + parser.add_argument( + "--pixsize", + type=float, + default=0.8, + required=False, + help="Size of a spectrum pixel in Angstrom, used to" + "define the binning of the output wavenumber array", + ) + + parser.add_argument( + "--weight-method", + type=str, + default="no_weights", + help="SNR weighting scheme for the mean P1D computation," + "Possible options: no_weights, fit_snr", + ) + + parser.add_argument( + "--apply_z_weights", + action="store_true", + default=False, + required=False, + help="If set, apply a z weighting scheme analog to that used for QMLE (eg. 2008.06421). " + "Each chunk with mean redshift z contributes two nearest bins z_i (resp. z_j=z_i+dz), " + "with weights |z-z_j|/dz (resp. |z-z_i|/dz).", + ) + + parser.add_argument( + "--output-snrfit", + type=str, + default=None, + help="Name of the ASCII file where SNR fit results are stored," + "if weight-method is fit_snr", + ) + + parser.add_argument( + "--snr-cut-scheme", + type=str, + default=None, + help="Choice of SNR cut type, " + "Possible options: eboss (varying cut vs z, as in eBOSS - DR14); fixed;" + "None (obligatory when --weight-method != no_weights)", + ) + + parser.add_argument( + "--snrcut", + type=float, + default=None, + help="Value of the SNR cut if snr-cut-scheme=fixed", + ) + + parser.add_argument( + "--skymask-file", + type=str, + default=None, + help="Name of the ASCII file containing skyline masks, which was used to" + "compute the deltas. If set, a correction for this mask will be computed," + 'with "*_skycorr" columns in the output.', + ) + + parser.add_argument( + "--minwave-skymask", + type=float, + default=utils.DEFAULT_MINWAVE_SKYMASK, + help="Parameter for the wavelength grid used for the skymask," + "if --skymask-file is set.", + ) + + parser.add_argument( + "--maxwave-skymask", + type=float, + default=utils.DEFAULT_MAXWAVE_SKYMASK, + help="Parameter for the wavelength grid used for the skymask," + "if --skymask-file is set.", + ) + + parser.add_argument( + "--dwave-skymask", + type=float, + default=utils.DEFAULT_DWAVE_SKYMASK, + help="Parameter for the wavelength grid used for the skymask," + "if --skymask-file is set.", + ) + + parser.add_argument( + "--overwrite", + action="store_true", + default=False, + required=False, + help="Overwrite the output", + ) + + parser.add_argument( + "--velunits", + action="store_true", + default=False, + required=False, + help="Compute mean P1D in velocity units", + ) + + parser.add_argument( + "--no-median", + action="store_true", + default=False, + required=False, + help="Skip computation of median quantities", + ) + + parser.add_argument( + "--ncpu", + type=int, + default=8, + required=False, + help="Number of CPUs used to read input P1D files", + ) + + parser.add_argument( + "--covariance", + action="store_true", + default=False, + required=False, + help="Compute covariance matrix", + ) + + parser.add_argument( + "--bootstrap", + action="store_true", + default=False, + required=False, + help="Compute covariance matrix with bootstrap method", + ) + + parser.add_argument( + "--nbootstrap", + type=int, + default=20, + required=False, + help="Number of bootstrap iteration used", + ) + + parser.add_argument( + "--bootstrap_average", + action="store_true", + default=False, + required=False, + help="Compute covariance matrix with bootstrap method on the average P1D", + ) + + parser.add_argument( + "--nbootstrap_average", + type=int, + default=1000, + required=False, + help="Number of bootstrap iteration used for average bootstrap", + ) + args = parser.parse_args(cmdargs) - if (args.weight_method != 'no_weights') and (args.snr_cut_scheme is not None): - raise ValueError("""You are using a weighting method with a + if (args.weight_method != "no_weights") and (args.snr_cut_scheme is not None): + raise ValueError( + """You are using a weighting method with a redshift-dependent SNR quality cut, this is not - tested and should bias the result""") - - if args.snr_cut_scheme == 'eboss': - snrcut = np.array([4.1, 3.9, 3.6, 3.2, 2.9, 2.6, 2.2, 2.0, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, - 2.0, 2.0, 2.0]) + tested and should bias the result""" + ) + + if args.snr_cut_scheme == "eboss": + snrcut = np.array( + [ + 4.1, + 3.9, + 3.6, + 3.2, + 2.9, + 2.6, + 2.2, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + 2.0, + ] + ) zbins_snrcut = np.arange(2.2, 6.4, 0.2) - elif args.snr_cut_scheme == 'fixed': + elif args.snr_cut_scheme == "fixed": snrcut = np.array([args.snrcut]) zbins_snrcut = None elif args.snr_cut_scheme == None: snrcut = None zbins_snrcut = None else: - raise ValueError("Unknown value for option --snr-cut-scheme" - "You may add here in the code a specific SNR cutting scheme") - - kedge_min, kedge_max, kedge_bin = define_wavenumber_array(args.kedge_min, - args.kedge_max, - args.kedge_bin, - args.velunits, - args.pixsize, - args.rebinfac) + raise ValueError( + "Unknown value for option --snr-cut-scheme" + "You may add here in the code a specific SNR cutting scheme" + ) + + kedge_min, kedge_max, kedge_bin = define_wavenumber_array( + args.kedge_min, + args.kedge_max, + args.kedge_bin, + args.velunits, + args.pixsize, + args.rebinfac, + ) k_edges = np.arange(kedge_min, kedge_max, kedge_bin) z_edges = np.around(np.arange(args.zedge_min, args.zedge_max, args.zedge_bin), 5) @@ -261,35 +353,47 @@ def main(cmdargs=None): snr_ext = "_snr_cut" if snrcut is not None else "" vel_ext = "_vel" if args.velunits else "" rebin_ext = f"_rebin{args.rebinfac}" if args.rebinfac != 1 else "" - output_file = os.path.join(args.in_dir, - f'mean_Pk1d_{args.weight_method}{med_ext}{snr_ext}{vel_ext}{rebin_ext}.fits.gz') + output_file = os.path.join( + args.in_dir, + f"mean_Pk1d_{args.weight_method}{med_ext}{snr_ext}{vel_ext}{rebin_ext}.fits.gz", + ) else: output_file = args.output_file if args.skymask_file is not None: - skymask_matrices = utils.skyline_mask_matrices_desi(z_edges, args.skymask_file, - minwave=args.minwave_skymask, - maxwave=args.maxwave_skymask, - dwave=args.dwave_skymask) + skymask_matrices = utils.skyline_mask_matrices_desi( + z_edges, + args.skymask_file, + minwave=args.minwave_skymask, + maxwave=args.maxwave_skymask, + dwave=args.dwave_skymask, + ) else: skymask_matrices = None - postproc_pk1d.run_postproc_pk1d(args.in_dir, output_file, - z_edges, - k_edges, - weight_method=args.weight_method, - apply_z_weights=args.apply_z_weights, - output_snrfit=args.output_snrfit, - snrcut=snrcut, - skymask_matrices=skymask_matrices, - zbins_snrcut=zbins_snrcut, - nomedians=args.no_median, - velunits=args.velunits, - overwrite=args.overwrite, - ncpu = args.ncpu, - compute_covariance=args.covariance, - compute_bootstrap=args.bootstrap, - number_bootstrap=args.nbootstrap, - compute_bootstrap_average=args.bootstrap_average, - number_bootstrap_average=args.nbootstrap_average, - ) + postproc_pk1d.run_postproc_pk1d( + args.in_dir, + output_file, + z_edges, + k_edges, + weight_method=args.weight_method, + apply_z_weights=args.apply_z_weights, + output_snrfit=args.output_snrfit, + snrcut=snrcut, + skymask_matrices=skymask_matrices, + zbins_snrcut=zbins_snrcut, + nomedians=args.no_median, + velunits=args.velunits, + overwrite=args.overwrite, + ncpu=args.ncpu, + compute_covariance=args.covariance, + compute_bootstrap=args.bootstrap, + number_bootstrap=args.nbootstrap, + compute_bootstrap_average=args.bootstrap_average, + number_bootstrap_average=args.nbootstrap_average, + ) + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_cf.py b/py/picca/bin/picca_cf.py old mode 100644 new mode 100755 index 87ce33255..cc7110785 --- a/py/picca/bin/picca_cf.py +++ b/py/picca/bin/picca_cf.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the auto and cross-correlation of delta fields. This module follow the procedure described in sections 3.1 and 3.2 of du Mas des @@ -6,8 +6,9 @@ """ import argparse import multiprocessing +import sys import time -from multiprocessing import Lock, Pool, Value, cpu_count +from multiprocessing import Lock, Value, cpu_count import fitsio import numpy as np @@ -95,18 +96,19 @@ def main(cmdargs=None): ) parser.add_argument( - '--nt', + "--nt", type=int, default=50, required=False, - help='Number of r-transverse bins', + help="Number of r-transverse bins", ) parser.add_argument( - '--rmu-binning', + "--rmu-binning", action="store_true", - help=('Estimate in r,mu binning. np becomes mu bins.' - ' nt becomes r bins. rp min max is always 0, 1' + help=( + "Estimate in r,mu binning. np becomes mu bins." + " nt becomes r bins. rp min max is always 0, 1" ), ) @@ -484,72 +486,85 @@ def main(cmdargs=None): num_pairs = num_pairs_list.sum(axis=0) # save data - results = fitsio.FITS(args.out, 'rw', clobber=True) - header = [{ - 'name': 'RPMIN', - 'value': cf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RPMAX', - 'value': cf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RTMAX', - 'value': cf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' - }, { - 'name': 'NP', - 'value': cf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' - }, { - 'name': 'NT', - 'value': cf.num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' - }, { - 'name': 'ZCUTMIN', - 'value': cf.z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, { - 'name': 'ZCUTMAX', - 'value': cf.z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, { - 'name': 'NSIDE', - 'value': cf.nside, - 'comment': 'Healpix nside' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - }, { - 'name': "RMU_BIN", - 'value': cf.rmu_binning, - 'comment': 'True if binned in r, mu' - } + results = fitsio.FITS(args.out, "rw", clobber=True) + header = [ + { + "name": "RPMIN", + "value": cf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": cf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", + }, + { + "name": "RTMAX", + "value": cf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", + }, + { + "name": "NP", + "value": cf.num_bins_r_par, + "comment": "Number of bins in r-parallel", + }, + { + "name": "NT", + "value": cf.num_bins_r_trans, + "comment": "Number of bins in r-transverse", + }, + { + "name": "ZCUTMIN", + "value": cf.z_cut_min, + "comment": "Minimum redshift of pairs", + }, + { + "name": "ZCUTMAX", + "value": cf.z_cut_max, + "comment": "Maximum redshift of pairs", + }, + {"name": "NSIDE", "value": cf.nside, "comment": "Healpix nside"}, + { + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + { + "name": "RMU_BIN", + "value": cf.rmu_binning, + "comment": "True if binned in r, mu", + }, ] results.write( [r_par, r_trans, z, num_pairs], - names=['RP', 'RT', 'Z', 'NB'], - comment=['R-parallel' if not cf.rmu_binning else 'Mu', - 'R-transverse' if not cf.rmu_binning else 'Radial separation', - 'Redshift', 'Number of pairs'], - units=['h^-1 Mpc' if not cf.rmu_binning else '', 'h^-1 Mpc', '', ''], + names=["RP", "RT", "Z", "NB"], + comment=[ + "R-parallel" if not cf.rmu_binning else "Mu", + "R-transverse" if not cf.rmu_binning else "Radial separation", + "Redshift", + "Number of pairs", + ], + units=["h^-1 Mpc" if not cf.rmu_binning else "", "h^-1 Mpc", "", ""], header=header, extname="ATTRI", ) @@ -568,3 +583,8 @@ def main(cmdargs=None): t3 = time.time() userprint(f"picca_cf.py - Time total : {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_cf1d.py b/py/picca/bin/picca_cf1d.py old mode 100644 new mode 100755 index 654786dac..7b3a706fd --- a/py/picca/bin/picca_cf1d.py +++ b/py/picca/bin/picca_cf1d.py @@ -1,15 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the 1D auto or cross-correlation between delta field from the same forest. """ -import sys import argparse import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value -import numpy as np +import sys +from multiprocessing import Lock, Value, cpu_count + import fitsio +import numpy as np -from picca import constants, cf, io +from picca import cf, constants, io from picca.utils import userprint @@ -38,117 +39,136 @@ def main(cmdargs): forest.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the 1D auto or cross-correlation between delta ' - 'field from the same forest.')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--in-dir2', - type=str, - default=None, - required=False, - help='Directory to 2nd delta files') - - parser.add_argument('--lambda-min', - type=float, - default=None, - required=False, - help='Lower limit on observed wavelength [Angstrom] (default: minimum available in data)') - - parser.add_argument('--lambda-max', - type=float, - default=None, - required=False, - help='Upper limit on observed wavelength [Angstrom] (default: maximum available in data)') - - parser.add_argument('--z-min-sources', - type=float, - default=0., - required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) - - parser.add_argument('--z-max-sources', - type=float, - default=10., - required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) - - parser.add_argument('--dll', - type=float, - default=3.e-4, - required=False, - help='Loglam bin size') + description=( + "Compute the 1D auto or cross-correlation between delta " + "field from the same forest." + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) parser.add_argument( - '--lambda-abs', + "--in-dir", type=str, - default='LYA', + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--in-dir2", + type=str, + default=None, required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help="Directory to 2nd delta files", + ) parser.add_argument( - '--lambda-abs2', + "--lambda-min", + type=float, + default=None, + required=False, + help="Lower limit on observed wavelength [Angstrom] (default: minimum available in data)", + ) + + parser.add_argument( + "--lambda-max", + type=float, + default=None, + required=False, + help="Upper limit on observed wavelength [Angstrom] (default: maximum available in data)", + ) + + parser.add_argument( + "--z-min-sources", + type=float, + default=0.0, + required=False, + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) + + parser.add_argument( + "--z-max-sources", + type=float, + default=10.0, + required=False, + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) + + parser.add_argument( + "--dll", type=float, default=3.0e-4, required=False, help="Loglam bin size" + ) + + parser.add_argument( + "--lambda-abs", + type=str, + default="LYA", + required=False, + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) + + parser.add_argument( + "--lambda-abs2", type=str, default=None, required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the 2nd delta (if not give, same as 1st delta)')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the 2nd delta (if not give, same as 1st delta)" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol', + "--z-evol", type=float, - default=1., + default=1.0, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol2', + "--z-evol2", type=float, - default=1., + default=1.0, + required=False, + help="Exponent of the redshift evolution of the 2nd delta field", + ) + + parser.add_argument( + "--no-project", + action="store_true", required=False, - help='Exponent of the redshift evolution of the 2nd delta field') - - parser.add_argument('--no-project', - action='store_true', - required=False, - help='Do not project out continuum fitting modes') - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') + help="Do not project out continuum fitting modes", + ) + + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) + + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, + required=False, + help="Maximum number of spectra to read", + ) args = parser.parse_args(cmdargs) @@ -162,7 +182,7 @@ def main(cmdargs): if args.lambda_max is not None: cf.log_lambda_max = np.log10(args.lambda_max) cf.delta_log_lambda = args.dll - + cf.x_correlation = False cf.lambda_abs = constants.ABSORBER_IGM[args.lambda_abs] @@ -172,16 +192,18 @@ def main(cmdargs): cf.lambda_abs2 = constants.ABSORBER_IGM[args.lambda_abs] ### Read data 1 - data, num_data, z_min, z_max = io.read_deltas(args.in_dir, - cf.nside, - cf.lambda_abs, - args.z_evol, - args.z_ref, - cosmo=None, - max_num_spec=args.nspec, - no_project=args.no_project, - z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + data, num_data, z_min, z_max = io.read_deltas( + args.in_dir, + cf.nside, + cf.lambda_abs, + args.z_evol, + args.z_ref, + cosmo=None, + max_num_spec=args.nspec, + no_project=args.no_project, + z_min_qso=args.z_min_sources, + z_max_qso=args.z_max_sources, + ) cf.data = data cf.num_data = num_data del z_min, z_max @@ -201,7 +223,8 @@ def main(cmdargs): max_num_spec=args.nspec, no_project=args.no_project, z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + z_max_qso=args.z_max_sources, + ) cf.data2 = data2 cf.num_data2 = num_data2 del z_min2, z_max2 @@ -217,56 +240,61 @@ def main(cmdargs): args.z_ref, cosmo=None, max_num_spec=args.nspec, - no_project=args.no_project) + no_project=args.no_project, + ) cf.data2 = data2 cf.num_data2 = num_data2 del z_min2, z_max2 - - #here we are trying to guess reasonable lambda bounds from the data + # here we are trying to guess reasonable lambda bounds from the data if args.lambda_min is None: - cf.log_lambda_min=np.min([d.log_lambda[0] for hp in cf.data.values() for d in hp]) + cf.log_lambda_min = np.min( + [d.log_lambda[0] for hp in cf.data.values() for d in hp] + ) if args.in_dir2: - log_lambda_min2=np.min([d.log_lambda[0] for hp in cf.data2.values() for d in hp]) - cf.log_lambda_min=np.min([cf.log_lambda_min,log_lambda_min2]) + log_lambda_min2 = np.min( + [d.log_lambda[0] for hp in cf.data2.values() for d in hp] + ) + cf.log_lambda_min = np.min([cf.log_lambda_min, log_lambda_min2]) print(f"lambda_min={10**cf.log_lambda_min:.3f}") if args.lambda_max is None: - cf.log_lambda_max=np.max([d.log_lambda[-1] for hp in cf.data.values() for d in hp]) + cf.log_lambda_max = np.max( + [d.log_lambda[-1] for hp in cf.data.values() for d in hp] + ) if args.in_dir2: - log_lambda_max2=np.max([d.log_lambda[-1] for hp in cf.data2.values() for d in hp]) - cf.log_lambda_max=np.max([cf.log_lambda_max,log_lambda_max2]) + log_lambda_max2 = np.max( + [d.log_lambda[-1] for hp in cf.data2.values() for d in hp] + ) + cf.log_lambda_max = np.max([cf.log_lambda_max, log_lambda_max2]) print(f"lambda_max={10**cf.log_lambda_max:.3f}") - - - cf.num_pixels = int((cf.log_lambda_max - cf.log_lambda_min) / - cf.delta_log_lambda + 1 + 0.5) #+1e-7 to fix cases where the division is an integer, but numerically slightly below that int, - + cf.num_pixels = int( + (cf.log_lambda_max - cf.log_lambda_min) / cf.delta_log_lambda + 1 + 0.5 + ) # +1e-7 to fix cases where the division is an integer, but numerically slightly below that int, # Convert lists to arrays cf.data = {key: np.array(value) for key, value in cf.data.items()} if cf.x_correlation: cf.data2 = {key: np.array(value) for key, value in cf.data2.items()} - # Compute the correlation function, use pool to parallelize - cf.counter = Value('i', 0) + cf.counter = Value("i", 0) cf.lock = Lock() - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") if cf.x_correlation: - healpixs = sorted([ - key for key in list(cf.data.keys()) if key in list(cf.data2.keys()) - ]) + healpixs = sorted( + [key for key in list(cf.data.keys()) if key in list(cf.data2.keys())] + ) else: healpixs = sorted(list(cf.data.keys())) - if args.nproc>1: + if args.nproc > 1: with context.Pool(processes=args.nproc) as pool: correlation_function_data = pool.map(corr_func, healpixs) else: correlation_function_data = [corr_func(h) for h in healpixs] - userprint('\n') + userprint("\n") # group data from parallelisation correlation_function_data = np.array(correlation_function_data) @@ -319,11 +347,11 @@ def main(cmdargs): num_pairs_list = num_pairs_list[w] rebin = np.bincount(dbin, weights=xi * weights_list) - xi_1d[:len(rebin)] = rebin + xi_1d[: len(rebin)] = rebin rebin = np.bincount(dbin, weights=weights_list) - weights_1d[:len(rebin)] = rebin + weights_1d[: len(rebin)] = rebin rebin = np.bincount(dbin, weights=num_pairs_list) - num_pairs1d[:len(rebin)] = rebin + num_pairs1d[: len(rebin)] = rebin w = weights_1d > 0 xi_1d[w] /= weights_1d[w] @@ -331,48 +359,59 @@ def main(cmdargs): # Save results userprint("writing") - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = [ { - 'name': 'LLMIN', - 'value': cf.log_lambda_min, - 'comment': 'Minimum log10 lambda [log Angstrom]' + "name": "LLMIN", + "value": cf.log_lambda_min, + "comment": "Minimum log10 lambda [log Angstrom]", }, { - 'name': 'LLMAX', - 'value': cf.log_lambda_max, - 'comment': 'Maximum log10 lambda [log Angstrom]' + "name": "LLMAX", + "value": cf.log_lambda_max, + "comment": "Maximum log10 lambda [log Angstrom]", }, { - 'name': 'DLL', - 'value': cf.delta_log_lambda, - 'comment': 'Loglam bin size [log Angstrom]' + "name": "DLL", + "value": cf.delta_log_lambda, + "comment": "Loglam bin size [log Angstrom]", }, ] comment = [ - 'Variance', 'Sum of weight for variance', 'Sum of pairs for variance', - 'Correlation', 'Sum of weight for correlation', - 'Sum of pairs for correlation' + "Variance", + "Sum of weight for variance", + "Sum of pairs for variance", + "Correlation", + "Sum of weight for correlation", + "Sum of pairs for correlation", ] - results.write([ - variance_1d, weights_variance_1d, num_pairs_variance_1d, xi_1d, - weights_1d, num_pairs1d - ], - names=['v1d', 'wv1d', 'nv1d', 'c1d', 'nc1d', 'nb1d'], - header=header, - comment=comment, - extname='1DCOR') - - comment = ['Covariance', 'Sum of weight', 'Number of pairs'] - results.write([xi_list_2d, weights_list_2d, num_pairs_list_2d], - names=['DA', 'WE', 'NB'], - comment=comment, - extname='2DCOR') + results.write( + [ + variance_1d, + weights_variance_1d, + num_pairs_variance_1d, + xi_1d, + weights_1d, + num_pairs1d, + ], + names=["v1d", "wv1d", "nv1d", "c1d", "nc1d", "nb1d"], + header=header, + comment=comment, + extname="1DCOR", + ) + + comment = ["Covariance", "Sum of weight", "Number of pairs"] + results.write( + [xi_list_2d, weights_list_2d, num_pairs_list_2d], + names=["DA", "WE", "NB"], + comment=comment, + extname="2DCOR", + ) results.close() userprint("all done") -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_cf_angl.py b/py/picca/bin/picca_cf_angl.py old mode 100644 new mode 100755 index 6d912eaea..4866af485 --- a/py/picca/bin/picca_cf_angl.py +++ b/py/picca/bin/picca_cf_angl.py @@ -1,15 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the auto and cross-correlation of delta fields as a function of angle and wavelength ratio """ -import sys import argparse import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value -import numpy as np +import sys +from multiprocessing import Lock, Value, cpu_count + import fitsio +import numpy as np -from picca import constants, cf, io +from picca import cf, constants, io from picca.utils import userprint @@ -38,155 +39,179 @@ def main(cmdargs): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the auto and cross-correlation of delta fields ' - 'as a function of angle and wavelength ratio')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--in-dir2', - type=str, - default=None, - required=False, - help='Directory to 2nd delta files') - - parser.add_argument('--wr-min', - type=float, - default=1., - required=False, - help='Min of wavelength ratio') - - parser.add_argument('--wr-max', - type=float, - default=1.1, - required=False, - help='Max of wavelength ratio') - - parser.add_argument('--ang-max', - type=float, - default=0.02, - required=False, - help='Max angle (rad)') - - parser.add_argument('--np', - type=int, - default=50, - required=False, - help='Number of wavelength ratio bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of angular bins') + description=( + "Compute the auto and cross-correlation of delta fields " + "as a function of angle and wavelength ratio" + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--in-dir2", + type=str, + default=None, + required=False, + help="Directory to 2nd delta files", + ) + + parser.add_argument( + "--wr-min", + type=float, + default=1.0, + required=False, + help="Min of wavelength ratio", + ) + + parser.add_argument( + "--wr-max", + type=float, + default=1.1, + required=False, + help="Max of wavelength ratio", + ) + + parser.add_argument( + "--ang-max", type=float, default=0.02, required=False, help="Max angle (rad)" + ) parser.add_argument( - '--z-cut-min', + "--np", + type=int, + default=50, + required=False, + help="Number of wavelength ratio bins", + ) + + parser.add_argument( + "--nt", type=int, default=50, required=False, help="Number of angular bins" + ) + + parser.add_argument( + "--z-cut-min", type=float, - default=0., + default=0.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--z-cut-max', + "--z-cut-max", type=float, - default=10., + default=10.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) parser.add_argument( - '--z-min-sources', + "--z-min-sources", type=float, - default=0., + default=0.0, required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--z-max-sources', + "--z-max-sources", type=float, - default=10., + default=10.0, required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--lambda-abs', + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) parser.add_argument( - '--lambda-abs2', + "--lambda-abs2", type=str, default=None, required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the 2nd delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the 2nd delta" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol', + "--z-evol", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol2', + "--z-evol2", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the 2nd delta field') + help="Exponent of the redshift evolution of the 2nd delta field", + ) + + parser.add_argument( + "--no-project", + action="store_true", + required=False, + help="Do not project out continuum fitting modes", + ) + + parser.add_argument( + "--remove-same-half-plate-close-pairs", + action="store_true", + required=False, + help="Reject pairs in the first bin in r-parallel from same half plate", + ) - parser.add_argument('--no-project', - action='store_true', - required=False, - help='Do not project out continuum fitting modes') + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) parser.add_argument( - '--remove-same-half-plate-close-pairs', - action='store_true', + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, required=False, - help='Reject pairs in the first bin in r-parallel from same half plate') - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') + help="Maximum number of spectra to read", + ) args = parser.parse_args(cmdargs) @@ -211,16 +236,18 @@ def main(cmdargs): cf.remove_same_half_plate_close_pairs = args.remove_same_half_plate_close_pairs ### Read data 1 - data, num_data, z_min, z_max = io.read_deltas(args.in_dir, - cf.nside, - cf.lambda_abs, - cf.alpha, - cf.z_ref, - cosmo=None, - max_num_spec=args.nspec, - no_project=args.no_project, - z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + data, num_data, z_min, z_max = io.read_deltas( + args.in_dir, + cf.nside, + cf.lambda_abs, + cf.alpha, + cf.z_ref, + cosmo=None, + max_num_spec=args.nspec, + no_project=args.no_project, + z_min_qso=args.z_min_sources, + z_max_qso=args.z_max_sources, + ) cf.data = data cf.num_data = num_data del z_min, z_max @@ -248,7 +275,8 @@ def main(cmdargs): max_num_spec=args.nspec, no_project=args.no_project, z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + z_max_qso=args.z_max_sources, + ) cf.data2 = data2 cf.num_data2 = num_data2 del z_min2, z_max2 @@ -256,10 +284,10 @@ def main(cmdargs): userprint("done, npix = {}".format(len(data2))) # compute correlation function, use pool to parallelize - cf.counter = Value('i', 0) + cf.counter = Value("i", 0) cf.lock = Lock() cpu_data = {healpix: [healpix] for healpix in data} - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=args.nproc) correlation_function_data = pool.map(corr_func, sorted(cpu_data.values())) pool.close() @@ -275,7 +303,7 @@ def main(cmdargs): healpix_list = np.array(sorted(list(cpu_data.keys()))) # normalize values - w = (weights_list.sum(axis=0) > 0.) + w = weights_list.sum(axis=0) > 0.0 r_par = (r_par_list * weights_list).sum(axis=0) r_par[w] /= weights_list.sum(axis=0)[w] r_trans = (r_trans_list * weights_list).sum(axis=0) @@ -284,62 +312,54 @@ def main(cmdargs): z[w] /= weights_list.sum(axis=0)[w] num_pairs = num_pairs_list.sum(axis=0) - results = fitsio.FITS(args.out, 'rw', clobber=True) - header = [{ - 'name': 'RPMIN', - 'value': cf.r_par_min, - 'comment': 'Minimum wavelength ratio' - }, { - 'name': 'RPMAX', - 'value': cf.r_par_max, - 'comment': 'Maximum wavelength ratio' - }, { - 'name': 'RTMAX', - 'value': cf.r_trans_max, - 'comment': 'Maximum angle [rad]' - }, { - 'name': 'NP', - 'value': cf.num_bins_r_par, - 'comment': 'Number of bins in wavelength ratio' - }, { - 'name': 'NT', - 'value': cf.num_bins_r_trans, - 'comment': 'Number of bins in angle' - }, { - 'name': 'ZCUTMIN', - 'value': cf.z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, { - 'name': 'ZCUTMAX', - 'value': cf.z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, { - 'name': 'NSIDE', - 'value': cf.nside, - 'comment': 'Healpix nside' - }] + results = fitsio.FITS(args.out, "rw", clobber=True) + header = [ + {"name": "RPMIN", "value": cf.r_par_min, "comment": "Minimum wavelength ratio"}, + {"name": "RPMAX", "value": cf.r_par_max, "comment": "Maximum wavelength ratio"}, + {"name": "RTMAX", "value": cf.r_trans_max, "comment": "Maximum angle [rad]"}, + { + "name": "NP", + "value": cf.num_bins_r_par, + "comment": "Number of bins in wavelength ratio", + }, + { + "name": "NT", + "value": cf.num_bins_r_trans, + "comment": "Number of bins in angle", + }, + { + "name": "ZCUTMIN", + "value": cf.z_cut_min, + "comment": "Minimum redshift of pairs", + }, + { + "name": "ZCUTMAX", + "value": cf.z_cut_max, + "comment": "Maximum redshift of pairs", + }, + {"name": "NSIDE", "value": cf.nside, "comment": "Healpix nside"}, + ] results.write( [r_par, r_trans, z, num_pairs], - names=['RP', 'RT', 'Z', 'NB'], - units=['', 'rad', '', ''], - comment=['Wavelength ratio', 'Angle', 'Redshift', 'Number of pairs'], + names=["RP", "RT", "Z", "NB"], + units=["", "rad", "", ""], + comment=["Wavelength ratio", "Angle", "Redshift", "Number of pairs"], header=header, - extname='ATTRI') - - header2 = [{ - 'name': 'HLPXSCHM', - 'value': 'RING', - 'comment': 'Healpix scheme' - }] - results.write([healpix_list, weights_list, xi_list], - names=['HEALPID', 'WE', 'DA'], - comment=['Healpix index', 'Sum of weight', 'Correlation'], - header=header2, - extname='COR') + extname="ATTRI", + ) + + header2 = [{"name": "HLPXSCHM", "value": "RING", "comment": "Healpix scheme"}] + results.write( + [healpix_list, weights_list, xi_list], + names=["HEALPID", "WE", "DA"], + comment=["Healpix index", "Sum of weight", "Correlation"], + header=header2, + extname="COR", + ) results.close() -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_co.py b/py/picca/bin/picca_co.py old mode 100644 new mode 100755 index b44b04e0c..71ceb53c3 --- a/py/picca/bin/picca_co.py +++ b/py/picca/bin/picca_co.py @@ -1,14 +1,14 @@ -#!/usr/bin/env python -"""Compute the auto and cross-correlation between catalogs of objects -""" +#!/usr/bin/env python3 +"""Compute the auto and cross-correlation between catalogs of objects""" import argparse -import sys import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value -import numpy as np +import sys +from multiprocessing import Lock, Value, cpu_count + import fitsio +import numpy as np -from picca import constants, co, io, utils +from picca import co, constants, io, utils from picca.utils import userprint @@ -36,157 +36,176 @@ def main(cmdargs): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the auto and cross-correlation between catalogs ' - 'of objects')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--drq', - type=str, - default=None, - required=True, - help='Catalog of objects in DRQ format') - - parser.add_argument('--drq2', - type=str, - default=None, - required=False, - help='Catalog of objects 2 in DRQ format') + description=( + "Compute the auto and cross-correlation between catalogs " "of objects" + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--drq", + type=str, + default=None, + required=True, + help="Catalog of objects in DRQ format", + ) + + parser.add_argument( + "--drq2", + type=str, + default=None, + required=False, + help="Catalog of objects 2 in DRQ format", + ) parser.add_argument( - '--mode', - type=str, - default='sdss', - choices=['sdss','desi'], - required=False, - help='type of catalog supplied, default sdss') - - parser.add_argument('--rp-min', - type=float, - default=0., - required=False, - help='Min r-parallel [h^-1 Mpc]') - - parser.add_argument('--rp-max', - type=float, - default=200., - required=False, - help='Max r-parallel [h^-1 Mpc]') - - parser.add_argument('--rt-max', - type=float, - default=200., - required=False, - help='Max r-transverse [h^-1 Mpc]') - - parser.add_argument('--np', - type=int, - default=50, - required=False, - help='Number of r-parallel bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of r-transverse bins') + "--mode", + type=str, + default="sdss", + choices=["sdss", "desi"], + required=False, + help="type of catalog supplied, default sdss", + ) + + parser.add_argument( + "--rp-min", + type=float, + default=0.0, + required=False, + help="Min r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rp-max", + type=float, + default=200.0, + required=False, + help="Max r-parallel [h^-1 Mpc]", + ) parser.add_argument( - '--z-cut-min', + "--rt-max", type=float, - default=0., + default=200.0, required=False, - help=('Use only pairs of object x object with the mean redshift larger ' - 'than z-cut-min')) + help="Max r-transverse [h^-1 Mpc]", + ) + + parser.add_argument( + "--np", type=int, default=50, required=False, help="Number of r-parallel bins" + ) + + parser.add_argument( + "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" + ) parser.add_argument( - '--z-cut-max', + "--z-cut-min", type=float, - default=10., + default=0.0, required=False, - help=('Use only pairs of object x object with the mean redshift lower ' - 'than z-cut-max')) - - parser.add_argument('--z-min-obj', - type=float, - default=0, - required=False, - help='Min redshift for object field') - - parser.add_argument('--z-max-obj', - type=float, - default=10, - required=False, - help='Max redshift for object field') - - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + help=( + "Use only pairs of object x object with the mean redshift larger " + "than z-cut-min" + ), + ) parser.add_argument( - '--z-evol-obj', + "--z-cut-max", type=float, - default=1., + default=10.0, required=False, - help='Exponent of the redshift evolution of the object field') + help=( + "Use only pairs of object x object with the mean redshift lower " + "than z-cut-max" + ), + ) parser.add_argument( - '--z-evol-obj2', + "--z-min-obj", type=float, - default=1., + default=0, required=False, - help='Exponent of the redshift evolution of the object 2 field') + help="Min redshift for object field", + ) parser.add_argument( - '--fid-Om', + "--z-max-obj", + type=float, + default=10, + required=False, + help="Max redshift for object field", + ) + + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) + + parser.add_argument( + "--z-evol-obj", + type=float, + default=1.0, + required=False, + help="Exponent of the redshift evolution of the object field", + ) + + parser.add_argument( + "--z-evol-obj2", + type=float, + default=1.0, + required=False, + help="Exponent of the redshift evolution of the object 2 field", + ) + + parser.add_argument( + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--type-corr', + "--type-corr", type=str, - default='DD', + default="DD", required=False, - help='type of correlation: DD, RR, DR, RD, xDD, xRR, xD1R2, xR1D2') + help="type of correlation: DD, RR, DR, RD, xDD, xRR, xD1R2, xR1D2", + ) - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) args = parser.parse_args(cmdargs) @@ -203,11 +222,13 @@ def main(cmdargs): co.num_bins_r_trans = args.nt co.nside = args.nside co.type_corr = args.type_corr - if co.type_corr not in [ - 'DD', 'RR', 'DR', 'RD', 'xDD', 'xRR', 'xD1R2', 'xR1D2' - ]: - userprint(("ERROR: type-corr not in ['DD', 'RR', 'DR', 'RD', 'xDD', " - "'xRR', 'xD1R2', 'xR1D2']")) + if co.type_corr not in ["DD", "RR", "DR", "RD", "xDD", "xRR", "xD1R2", "xR1D2"]: + userprint( + ( + "ERROR: type-corr not in ['DD', 'RR', 'DR', 'RD', 'xDD', " + "'xRR', 'xD1R2', 'xR1D2']" + ) + ) sys.exit() if args.drq2 is None: co.x_correlation = False @@ -215,15 +236,21 @@ def main(cmdargs): co.x_correlation = True # load fiducial cosmology - cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl) + cosmo = constants.Cosmo( + Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, wl=args.fid_wl + ) ### Read objects 1 - objs, z_min = io.read_objects(args.drq, args.nside, args.z_min_obj, - args.z_max_obj, args.z_evol_obj, args.z_ref, - cosmo,mode=args.mode) + objs, z_min = io.read_objects( + args.drq, + args.nside, + args.z_min_obj, + args.z_max_obj, + args.z_evol_obj, + args.z_ref, + cosmo, + mode=args.mode, + ) userprint("") co.objs = objs co.num_data = len([obj for healpix in co.objs for obj in co.objs[healpix]]) @@ -232,22 +259,28 @@ def main(cmdargs): ### Read objects 2 if co.x_correlation: - objs2, z_min2 = io.read_objects(args.drq2, args.nside, args.z_min_obj, - args.z_max_obj, args.z_evol_obj2, - args.z_ref, cosmo,mode=args.mode) + objs2, z_min2 = io.read_objects( + args.drq2, + args.nside, + args.z_min_obj, + args.z_max_obj, + args.z_evol_obj2, + args.z_ref, + cosmo, + mode=args.mode, + ) userprint("") co.objs2 = objs2 # recompute maximum angular separation co.ang_max = utils.compute_ang_max(cosmo, co.r_trans_max, z_min, z_min2) # compute correlation function, use pool to parallelize - co.counter = Value('i', 0) + co.counter = Value("i", 0) co.lock = Lock() cpu_data = {healpix: [healpix] for healpix in co.objs} - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=args.nproc) - correlation_function_data = pool.map(corr_func, - sorted(list(cpu_data.values()))) + correlation_function_data = pool.map(corr_func, sorted(list(cpu_data.values()))) pool.close() # group data from parallelisation @@ -259,7 +292,7 @@ def main(cmdargs): num_pairs_list = correlation_function_data[:, 4, :].astype(np.int64) healpix_list = np.array(sorted(list(cpu_data))) - w = (weights_list.sum(axis=0) > 0.) + w = weights_list.sum(axis=0) > 0.0 r_par = (r_par_list * weights_list).sum(axis=0) r_par[w] /= weights_list.sum(axis=0)[w] r_trans = (r_trans_list * weights_list).sum(axis=0) @@ -268,105 +301,97 @@ def main(cmdargs): z[w] /= weights_list.sum(axis=0)[w] num_pairs = num_pairs_list.sum(axis=0) - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = [ { - 'name': 'RPMIN', - 'value': co.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' + "name": "RPMIN", + "value": co.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", }, { - 'name': 'RPMAX', - 'value': co.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' + "name": "RPMAX", + "value": co.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", }, { - 'name': 'RTMAX', - 'value': co.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' + "name": "RTMAX", + "value": co.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", }, { - 'name': 'NP', - 'value': co.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' + "name": "NP", + "value": co.num_bins_r_par, + "comment": "Number of bins in r-parallel", }, { - 'name': 'NT', - 'value': co.num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' + "name": "NT", + "value": co.num_bins_r_trans, + "comment": "Number of bins in r-transverse", }, + {"name": "NSIDE", "value": co.nside, "comment": "Healpix nside"}, + {"name": "TYPECORR", "value": co.type_corr, "comment": "Correlation type"}, { - 'name': 'NSIDE', - 'value': co.nside, - 'comment': 'Healpix nside' + "name": "NOBJ", + "value": len([obj for healpix in co.objs for obj in co.objs[healpix]]), + "comment": "Number of objects", }, { - 'name': 'TYPECORR', - 'value': co.type_corr, - 'comment': 'Correlation type' + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': - 'NOBJ', - 'value': - len([obj for healpix in co.objs for obj in co.objs[healpix]]), - 'comment': - 'Number of objects' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - } - ] + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + ] if co.x_correlation: - header += [{ - 'name': - 'NOBJ2', - 'value': - len([ - obj2 for healpix in co.objs2 for obj2 in co.objs2[healpix] - ]), - 'comment': - 'Number of objects 2' - }] - - comment = ['R-parallel', 'R-transverse', 'Redshift', 'Number of pairs'] - units = ['h^-1 Mpc', 'h^-1 Mpc', '', ''] - results.write([r_par, r_trans, z, num_pairs], - names=['RP', 'RT', 'Z', 'NB'], - header=header, - comment=comment, - units=units, - extname='ATTRI') - - comment = ['Healpix index', 'Sum of weight', 'Number of pairs'] - header2 = [{ - 'name': 'HLPXSCHM', - 'value': 'RING', - 'comment': 'healpix scheme' - }] - results.write([healpix_list, weights_list, num_pairs_list], - names=['HEALPID', 'WE', 'NB'], - header=header2, - comment=comment, - extname='COR') + header += [ + { + "name": "NOBJ2", + "value": len( + [obj2 for healpix in co.objs2 for obj2 in co.objs2[healpix]] + ), + "comment": "Number of objects 2", + } + ] + + comment = ["R-parallel", "R-transverse", "Redshift", "Number of pairs"] + units = ["h^-1 Mpc", "h^-1 Mpc", "", ""] + results.write( + [r_par, r_trans, z, num_pairs], + names=["RP", "RT", "Z", "NB"], + header=header, + comment=comment, + units=units, + extname="ATTRI", + ) + + comment = ["Healpix index", "Sum of weight", "Number of pairs"] + header2 = [{"name": "HLPXSCHM", "value": "RING", "comment": "healpix scheme"}] + results.write( + [healpix_list, weights_list, num_pairs_list], + names=["HEALPID", "WE", "NB"], + header=header2, + comment=comment, + extname="COR", + ) results.close() userprint("\nFinished") -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_coadd_zint.py b/py/picca/bin/picca_coadd_zint.py old mode 100644 new mode 100755 index 007d7c24f..9ba4212ce --- a/py/picca/bin/picca_coadd_zint.py +++ b/py/picca/bin/picca_coadd_zint.py @@ -1,15 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Coadd correlation function from different redshift intervals""" +import argparse import os import sys -import argparse + import fitsio import numpy as np from picca.utils import userprint -def coadd_correlations(input_files,output_file): +def coadd_correlations(input_files, output_file): """Coadds correlation functions measured in different redshift intervals. Args: @@ -21,42 +22,42 @@ def coadd_correlations(input_files,output_file): # specify which header entries we want to check are consistent across all # files being coadded - headers_to_check_match = ['NP','NT','OMEGAM','OMEGAR','OMEGAK','WL','NSIDE'] + headers_to_check_match = ["NP", "NT", "OMEGAM", "OMEGAR", "OMEGAK", "WL", "NSIDE"] # initialize coadd arrays, fill them with zeros with fitsio.FITS(input_files[0]) as hdul: - r_par = hdul[1]['RP'][:] * 0 - r_trans = hdul[1]['RT'][:] * 0 - num_pairs = hdul[1]['NB'][:] * 0 - z = hdul[1]['Z'][:] * 0 + r_par = hdul[1]["RP"][:] * 0 + r_trans = hdul[1]["RT"][:] * 0 + num_pairs = hdul[1]["NB"][:] * 0 + z = hdul[1]["Z"][:] * 0 weights_total = r_par * 0 - healpixs = hdul[2]['HEALPID'][:] + healpixs = hdul[2]["HEALPID"][:] # get values of header entries to check with other files header = hdul[1].read_header() - headers_to_check_match_values = {h:header[h] for h in headers_to_check_match} + headers_to_check_match_values = {h: header[h] for h in headers_to_check_match} # initialise header quantities - r_par_min = 1.e6 - r_par_max = -1.e6 - r_trans_max = -1.e6 - z_cut_min = 1.e6 - z_cut_max = -1.e6 + r_par_min = 1.0e6 + r_par_max = -1.0e6 + r_trans_max = -1.0e6 + z_cut_min = 1.0e6 + z_cut_max = -1.0e6 # check to see if all files have the same HEALPix pixels same_healpixs = True for file in input_files: with fitsio.FITS(file) as hdul: - if len(hdul[2]['HEALPID'][:]) != len(healpixs): + if len(hdul[2]["HEALPID"][:]) != len(healpixs): same_healpixs = False - elif (hdul[2]['HEALPID'][:] != healpixs).all(): + elif (hdul[2]["HEALPID"][:] != healpixs).all(): same_healpixs = False # if so, initialise our xi and weights as arrays if same_healpixs: with fitsio.FITS(file) as hdul: - xi = hdul[2]['DA'][:] * 0 - weights = hdul[2]['WE'][:] * 0 + xi = hdul[2]["DA"][:] * 0 + weights = hdul[2]["WE"][:] * 0 # otherwise, use dictionaries to match HEALPix pixels carefully (but slower) else: xi = {} @@ -75,36 +76,36 @@ def coadd_correlations(input_files,output_file): assert header[entry] == headers_to_check_match_values[entry] # add weighted contributions from this file to coadd variables - weights_aux = hdul[2]['WE'][:] + weights_aux = hdul[2]["WE"][:] weights_total_aux = weights_aux.sum(axis=0) - r_par += hdul[1]['RP'][:] * weights_total_aux - r_trans += hdul[1]['RT'][:] * weights_total_aux - z += hdul[1]['Z'][:] * weights_total_aux - num_pairs += hdul[1]['NB'][:] + r_par += hdul[1]["RP"][:] * weights_total_aux + r_trans += hdul[1]["RT"][:] * weights_total_aux + z += hdul[1]["Z"][:] * weights_total_aux + num_pairs += hdul[1]["NB"][:] weights_total += weights_total_aux # update values to go in coadd header - r_par_min = np.min([r_par_min,header['RPMIN']]) - r_par_max = np.max([r_par_max,header['RPMAX']]) - r_trans_max = np.max([r_trans_max,header['RTMAX']]) - z_cut_min = np.min([z_cut_min,header['ZCUTMIN']]) - z_cut_max = np.max([z_cut_max,header['ZCUTMAX']]) + r_par_min = np.min([r_par_min, header["RPMIN"]]) + r_par_max = np.max([r_par_max, header["RPMAX"]]) + r_trans_max = np.max([r_trans_max, header["RTMAX"]]) + z_cut_min = np.min([z_cut_min, header["ZCUTMIN"]]) + z_cut_max = np.max([z_cut_max, header["ZCUTMAX"]]) # add xi and weights information to initialised structures if same_healpixs: xi += hdul[2]["DA"][:] * weights_aux weights += weights_aux else: - healpixs = hdul[2]['HEALPID'][:] + healpixs = hdul[2]["HEALPID"][:] for index, healpix in enumerate(healpixs): - userprint(" -> coadding healpix {}".format(healpix),end='\r') + userprint(" -> coadding healpix {}".format(healpix), end="\r") if healpix in xi: xi[healpix] += hdul[2]["DA"][:][index] * weights_aux[index] weights[healpix] += weights_aux[index, :] else: xi[healpix] = hdul[2]["DA"][:][index] * weights_aux[index] weights[healpix] = weights_aux[index] - userprint('') + userprint("") hdul.close() @@ -115,103 +116,95 @@ def coadd_correlations(input_files,output_file): weights = np.vstack([weights[healpix] for healpix in healpixs]) # normalise xi by the weights - w = weights>0 + w = weights > 0 xi[w] /= weights[w] # normalize all other quantities by total weights - w = weights_total>0 + w = weights_total > 0 r_par[w] /= weights_total[w] r_trans[w] /= weights_total[w] z[w] /= weights_total[w] - results = fitsio.FITS(output_file, 'rw', clobber=True) + results = fitsio.FITS(output_file, "rw", clobber=True) header = [ - { - 'name': 'RPMIN', - 'value': r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' - }, - { - 'name': 'RPMAX', - 'value': r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' - }, - { - 'name': 'RTMAX', - 'value': r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' - }, - { - 'name': 'NP', - 'value': headers_to_check_match_values['NP'], - 'comment': 'Number of bins in r-parallel' - }, - { - 'name': 'NT', - 'value': headers_to_check_match_values['NT'], - 'comment': 'Number of bins in r-transverse' - }, - { - 'name': 'ZCUTMIN', - 'value': z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, - { - 'name': 'ZCUTMAX', - 'value': z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, - { - 'name': 'NSIDE', - 'value': headers_to_check_match_values['NSIDE'], - 'comment': 'Healpix nside' - }, - { - 'name': 'OMEGAM', - 'value': headers_to_check_match_values['OMEGAM'], - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, - { - 'name': 'OMEGAR', - 'value': headers_to_check_match_values['OMEGAR'], - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, - { - 'name': 'OMEGAK', - 'value': headers_to_check_match_values['OMEGAK'], - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, - { - 'name': 'WL', - 'value': headers_to_check_match_values['WL'], - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - } + { + "name": "RPMIN", + "value": r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", + }, + { + "name": "RTMAX", + "value": r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", + }, + { + "name": "NP", + "value": headers_to_check_match_values["NP"], + "comment": "Number of bins in r-parallel", + }, + { + "name": "NT", + "value": headers_to_check_match_values["NT"], + "comment": "Number of bins in r-transverse", + }, + {"name": "ZCUTMIN", "value": z_cut_min, "comment": "Minimum redshift of pairs"}, + {"name": "ZCUTMAX", "value": z_cut_max, "comment": "Maximum redshift of pairs"}, + { + "name": "NSIDE", + "value": headers_to_check_match_values["NSIDE"], + "comment": "Healpix nside", + }, + { + "name": "OMEGAM", + "value": headers_to_check_match_values["OMEGAM"], + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": headers_to_check_match_values["OMEGAR"], + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": headers_to_check_match_values["OMEGAK"], + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": headers_to_check_match_values["WL"], + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, ] results.write( [r_par, r_trans, z, num_pairs], - names=['RP', 'RT', 'Z', 'NB'], - comment=['R-parallel', 'R-transverse', 'Redshift', 'Number of pairs'], - units=['h^-1 Mpc', 'h^-1 Mpc', '', ''], + names=["RP", "RT", "Z", "NB"], + comment=["R-parallel", "R-transverse", "Redshift", "Number of pairs"], + units=["h^-1 Mpc", "h^-1 Mpc", "", ""], header=header, - extname='ATTRI') - - header2 = [{ - 'name': 'HLPXSCHM', - 'value': 'RING', - 'comment': 'Healpix scheme' - }] - results.write([healpixs, weights, xi], - names=['HEALPID', 'WE', 'DA'], - comment=['Healpix index', 'Sum of weight', 'Correlation'], - header=header2, - extname='COR') + extname="ATTRI", + ) + + header2 = [{"name": "HLPXSCHM", "value": "RING", "comment": "Healpix scheme"}] + results.write( + [healpixs, weights, xi], + names=["HEALPID", "WE", "DA"], + comment=["Healpix index", "Sum of weight", "Correlation"], + header=header2, + extname="COR", + ) results.close() return -def coadd_dmats(input_files,output_file): + +def coadd_dmats(input_files, output_file): """Coadds distortion matrices measured in different redshift intervals. Args: @@ -223,29 +216,38 @@ def coadd_dmats(input_files,output_file): # specify which header entries we want to check are consistent across all # files being coadded - headers_to_check_match = ['NP','NT','COEFMOD','REJ','OMEGAM','OMEGAR','OMEGAK','WL'] + headers_to_check_match = [ + "NP", + "NT", + "COEFMOD", + "REJ", + "OMEGAM", + "OMEGAR", + "OMEGAK", + "WL", + ] # initialize distortion matrix array, fill them with zeros with fitsio.FITS(input_files[0]) as hdul: - dmat = hdul[1]['DM'][:] * 0 - weights_dmat = hdul[1]['WDM'][:] * 0 - r_par_dmat = np.zeros(hdul[2]['RP'][:].size) - r_trans_dmat = np.zeros(hdul[2]['RT'][:].size) - z_dmat = np.zeros(hdul[2]['Z'][:].size) - num_pairs_dmat = 0. + dmat = hdul[1]["DM"][:] * 0 + weights_dmat = hdul[1]["WDM"][:] * 0 + r_par_dmat = np.zeros(hdul[2]["RP"][:].size) + r_trans_dmat = np.zeros(hdul[2]["RT"][:].size) + z_dmat = np.zeros(hdul[2]["Z"][:].size) + num_pairs_dmat = 0.0 # get values of header entries to check with other files header = hdul[1].read_header() - headers_to_check_match_values = {h:header[h] for h in headers_to_check_match} + headers_to_check_match_values = {h: header[h] for h in headers_to_check_match} # initialise header quantities num_pairs = 0 num_pairs_used = 0 - r_par_min = 1.e6 - r_par_max = -1.e6 - r_trans_max = -1.e6 - z_cut_min = 1.e6 - z_cut_max = -1.e6 + r_par_min = 1.0e6 + r_par_max = -1.0e6 + r_trans_max = -1.0e6 + z_cut_min = 1.0e6 + z_cut_max = -1.0e6 # loop over files for file in input_files: @@ -260,173 +262,162 @@ def coadd_dmats(input_files,output_file): assert header[entry] == headers_to_check_match_values[entry] # add weighted contributions from this file to coadd variables - weights_dmat_aux = hdul[1]['WDM'][:] - dmat += hdul[1]['DM'][:] * weights_dmat_aux + weights_dmat_aux = hdul[1]["WDM"][:] + dmat += hdul[1]["DM"][:] * weights_dmat_aux weights_dmat += weights_dmat_aux - r_par_dmat += hdul[2]['RP'][:] * weights_dmat_aux - r_trans_dmat += hdul[2]['RT'][:] * weights_dmat_aux - z_dmat += hdul[2]['Z'][:] * weights_dmat_aux - num_pairs_dmat += 1. + r_par_dmat += hdul[2]["RP"][:] * weights_dmat_aux + r_trans_dmat += hdul[2]["RT"][:] * weights_dmat_aux + z_dmat += hdul[2]["Z"][:] * weights_dmat_aux + num_pairs_dmat += 1.0 # update values to go in coadd header - num_pairs += header['NPALL'] - num_pairs_used += header['NPUSED'] - r_par_min = np.min([r_par_min,header['RPMIN']]) - r_par_max = np.max([r_par_max,header['RPMAX']]) - r_trans_max = np.max([r_trans_max,header['RTMAX']]) - z_cut_min = np.min([z_cut_min,header['ZCUTMIN']]) - z_cut_max = np.max([z_cut_max,header['ZCUTMAX']]) + num_pairs += header["NPALL"] + num_pairs_used += header["NPUSED"] + r_par_min = np.min([r_par_min, header["RPMIN"]]) + r_par_max = np.max([r_par_max, header["RPMAX"]]) + r_trans_max = np.max([r_trans_max, header["RTMAX"]]) + z_cut_min = np.min([z_cut_min, header["ZCUTMIN"]]) + z_cut_max = np.max([z_cut_max, header["ZCUTMAX"]]) hdul.close() # normalize - w = weights_dmat>0 + w = weights_dmat > 0 dmat[w] /= weights_dmat[w] r_par_dmat[w] /= weights_dmat[w] r_trans_dmat[w] /= weights_dmat[w] z_dmat[w] /= weights_dmat[w] # save results - results = fitsio.FITS(output_file, 'rw', clobber=True) + results = fitsio.FITS(output_file, "rw", clobber=True) header = [ { - 'name': 'RPMIN', - 'value': r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' + "name": "RPMIN", + "value": r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", }, { - 'name': 'RPMAX', - 'value': r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' + "name": "RPMAX", + "value": r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", }, { - 'name': 'RTMAX', - 'value': r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' + "name": "RTMAX", + "value": r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", }, { - 'name': 'NP', - 'value': headers_to_check_match_values['NP'], - 'comment': 'Number of bins in r-parallel' + "name": "NP", + "value": headers_to_check_match_values["NP"], + "comment": "Number of bins in r-parallel", }, { - 'name': 'NT', - 'value': headers_to_check_match_values['NT'], - 'comment': 'Number of bins in r-transverse' + "name": "NT", + "value": headers_to_check_match_values["NT"], + "comment": "Number of bins in r-transverse", }, { - 'name': 'COEFMOD', - 'value': headers_to_check_match_values['COEFMOD'], - 'comment': 'Coefficient for model binning' + "name": "COEFMOD", + "value": headers_to_check_match_values["COEFMOD"], + "comment": "Coefficient for model binning", }, + {"name": "ZCUTMIN", "value": z_cut_min, "comment": "Minimum redshift of pairs"}, + {"name": "ZCUTMAX", "value": z_cut_max, "comment": "Maximum redshift of pairs"}, { - 'name': 'ZCUTMIN', - 'value': z_cut_min, - 'comment': 'Minimum redshift of pairs' + "name": "REJ", + "value": headers_to_check_match_values["REJ"], + "comment": "Rejection factor", }, + {"name": "NPALL", "value": num_pairs, "comment": "Number of pairs"}, + {"name": "NPUSED", "value": num_pairs_used, "comment": "Number of used pairs"}, { - 'name': 'ZCUTMAX', - 'value': z_cut_max, - 'comment': 'Maximum redshift of pairs' + "name": "OMEGAM", + "value": headers_to_check_match_values["OMEGAM"], + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'REJ', - 'value': headers_to_check_match_values['REJ'], - 'comment': 'Rejection factor' + "name": "OMEGAR", + "value": headers_to_check_match_values["OMEGAR"], + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'NPALL', - 'value': num_pairs, - 'comment': 'Number of pairs' + "name": "OMEGAK", + "value": headers_to_check_match_values["OMEGAK"], + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'NPUSED', - 'value': num_pairs_used, - 'comment': 'Number of used pairs' + "name": "WL", + "value": headers_to_check_match_values["WL"], + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", }, - { - 'name': 'OMEGAM', - 'value': headers_to_check_match_values['OMEGAM'], - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, - { - 'name': 'OMEGAR', - 'value': headers_to_check_match_values['OMEGAR'], - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, - { - 'name': 'OMEGAK', - 'value': headers_to_check_match_values['OMEGAK'], - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, - { - 'name': 'WL', - 'value': headers_to_check_match_values['WL'], - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - } - ] - results.write([weights_dmat, dmat], - names=['WDM', 'DM'], - comment=['Sum of weight', 'Distortion matrix'], - units=['', ''], - header=header, - extname='DMAT') - results.write([r_par_dmat, r_trans_dmat, z_dmat], - names=['RP', 'RT', 'Z'], - comment=['R-parallel', 'R-transverse', 'Redshift'], - units=['h^-1 Mpc', 'h^-1 Mpc', ''], - extname='ATTRI') + ] + results.write( + [weights_dmat, dmat], + names=["WDM", "DM"], + comment=["Sum of weight", "Distortion matrix"], + units=["", ""], + header=header, + extname="DMAT", + ) + results.write( + [r_par_dmat, r_trans_dmat, z_dmat], + names=["RP", "RT", "Z"], + comment=["R-parallel", "R-transverse", "Redshift"], + units=["h^-1 Mpc", "h^-1 Mpc", ""], + extname="ATTRI", + ) results.close() return + def main(cmdargs): """Coadds correlation function from different redshift intervals""" parser = argparse.ArgumentParser() - parser.add_argument("--data", - type=str, - nargs="*", - required=False, - help="the (x)cf....fits files to be coadded") + parser.add_argument( + "--data", + type=str, + nargs="*", + required=False, + help="the (x)cf....fits files to be coadded", + ) - parser.add_argument("--out", - type=str, - required=False, - help="name of output file") + parser.add_argument("--out", type=str, required=False, help="name of output file") - parser.add_argument("--dmats", - type=str, - nargs="*", - required=False, - help="the (x)dmat....fits files to be coadded.") + parser.add_argument( + "--dmats", + type=str, + nargs="*", + required=False, + help="the (x)dmat....fits files to be coadded.", + ) - parser.add_argument("--out-dmat", - type=str, - required=False, - help="name of dmat output file") + parser.add_argument( + "--out-dmat", type=str, required=False, help="name of dmat output file" + ) args = parser.parse_args(cmdargs) if args.data is None and args.dmats is None: - raise IOError('No input correlations or dmats provided!') + raise IOError("No input correlations or dmats provided!") if args.data is not None: for file in args.data: if not os.path.isfile(file): - userprint('WARN: could not find file {}, removing it'.format(file)) + userprint("WARN: could not find file {}, removing it".format(file)) args.data.remove(file) - coadd_correlations(args.data,args.out) + coadd_correlations(args.data, args.out) if args.dmats is not None: for file in args.dmats: if not os.path.isfile(file): - userprint('WARN: could not find file {}, removing it'.format(file)) + userprint("WARN: could not find file {}, removing it".format(file)) args.data.remove(file) - coadd_dmats(args.dmats,args.out_dmat) + coadd_dmats(args.dmats, args.out_dmat) if __name__ == "__main__": - cmdargs=sys.argv[1:] + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_compute_fvoigt.py b/py/picca/bin/picca_compute_fvoigt.py old mode 100644 new mode 100755 index da81b16b6..29596288a --- a/py/picca/bin/picca_compute_fvoigt.py +++ b/py/picca/bin/picca_compute_fvoigt.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import sys import numpy as np from scipy.constants import speed_of_light @@ -10,7 +11,7 @@ def voigt_profile(x, sigma=1, gamma=1): """Return the Voigt line shape at x with Lorentzian component HWHM gamma""" - return np.real(wofz((x + 1j*gamma) / (sigma * np.sqrt(2)))) + return np.real(wofz((x + 1j * gamma) / (sigma * np.sqrt(2)))) def get_voigt_profile_wave(wave, z, N_hi): @@ -49,10 +50,10 @@ def get_voigt_profile_wave(wave, z, N_hi): a = gamma / (4 * np.pi * Deltat_wave) * wave_lya**2 / c * 1e-10 u = (wave_rf - wave_lya) / Deltat_wave - H = voigt_profile(u, np.sqrt(1/2), a) + H = voigt_profile(u, np.sqrt(1 / 2), a) absorbtion = np.sqrt(np.pi) * e**2 * f * wave_lya**2 * 1e-10 * H - absorbtion /= (4 * np.pi * epsilon0 * me * c**2 * Deltat_wave) + absorbtion /= 4 * np.pi * epsilon0 * me * c**2 * Deltat_wave # 10^N_hi in cm^-2 and absorb in m^2 tau = (10**N_hi * 1e4 * absorbtion).astype(float) @@ -86,9 +87,13 @@ def profile_wave_to_comov_dist(wave, profile_wave, cosmo, differential=False): if differential: # We are in the f(lambda)dlambda = f(r)dr case, # so need to account for dlambda / dr = H(z) * Lambda_Lya / c - profile_comov_dist = cosmo.get_hubble(z) * constants.ABSORBER_IGM["LYA"] / speed_of_light + profile_comov_dist = ( + cosmo.get_hubble(z) * constants.ABSORBER_IGM["LYA"] / speed_of_light + ) - profile_comov_dist = np.interp(lin_spaced_comov_dist, comov_dist, profile_comov_dist) + profile_comov_dist = np.interp( + lin_spaced_comov_dist, comov_dist, profile_comov_dist + ) return lin_spaced_comov_dist, profile_comov_dist @@ -117,30 +122,58 @@ def fft_profile(profile, dx): def main(): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute FVoigt profile')) - - parser.add_argument('-o', '--output', type=str, default=None, - help=('Name and path of file to write FVoigt profile to')) - - parser.add_argument('--weights', type=str, default=None, - help=('File with total weight as a function of observed wavelength')) - - parser.add_argument('--cddf', type=str, default=None, - help=('File with column density distribution function (CDDF)')) - - parser.add_argument('--dla-prob', type=str, default=None, - help=('File with probability of finding DLAs in a forest ' - 'as a function of observed wavelength')) - - parser.add_argument('--z-dla', type=float, default=None, - help=('Mean DLA redshift')) - - parser.add_argument('--normalize', action='store_true', required=False, - help=('Whether the output FVoigt function is normalized')) - - parser.add_argument('--positive-fvoigt', action='store_true', required=False, - help=('Whether the output FVoigt function should be positive')) + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description=("Compute FVoigt profile"), + ) + + parser.add_argument( + "-o", + "--output", + type=str, + default=None, + help=("Name and path of file to write FVoigt profile to"), + ) + + parser.add_argument( + "--weights", + type=str, + default=None, + help=("File with total weight as a function of observed wavelength"), + ) + + parser.add_argument( + "--cddf", + type=str, + default=None, + help=("File with column density distribution function (CDDF)"), + ) + + parser.add_argument( + "--dla-prob", + type=str, + default=None, + help=( + "File with probability of finding DLAs in a forest " + "as a function of observed wavelength" + ), + ) + + parser.add_argument("--z-dla", type=float, default=None, help=("Mean DLA redshift")) + + parser.add_argument( + "--normalize", + action="store_true", + required=False, + help=("Whether the output FVoigt function is normalized"), + ) + + parser.add_argument( + "--positive-fvoigt", + action="store_true", + required=False, + help=("Whether the output FVoigt function should be positive"), + ) args = parser.parse_args() @@ -151,12 +184,15 @@ def main(): fidcosmo = constants.Cosmo(Om=0.3147) comov_dist_prob, dla_prob_comov_dist = profile_wave_to_comov_dist( - dla_prob_wave[0], dla_prob_wave[1], fidcosmo, differential=True) + dla_prob_wave[0], dla_prob_wave[1], fidcosmo, differential=True + ) comov_dist_weights, weights_comov_dist = profile_wave_to_comov_dist( - weights_wave[:, 0], weights_wave[:, 1], fidcosmo) + weights_wave[:, 0], weights_wave[:, 1], fidcosmo + ) weight_interp = np.interp( - comov_dist_prob, comov_dist_weights, weights_comov_dist, left=0, right=0) + comov_dist_prob, comov_dist_weights, weights_comov_dist, left=0, right=0 + ) mean_density = np.average(dla_prob_comov_dist, weights=weight_interp) wave = np.arange(2000, 8000, 1) # TODO this grid may be too sparse @@ -166,8 +202,12 @@ def main(): profile_wave /= np.mean(profile_wave) # r is in Mpc h^-1 --> k (from tf) will be in (Mpc h^-1)^-1 = h Mpc^-1 :) - comov_dist, profile_comov_dist = profile_wave_to_comov_dist(wave, profile_wave, fidcosmo) - k, fourier_profile = fft_profile(profile_comov_dist, np.abs(comov_dist[1] - comov_dist[0])) + comov_dist, profile_comov_dist = profile_wave_to_comov_dist( + wave, profile_wave, fidcosmo + ) + k, fourier_profile = fft_profile( + profile_comov_dist, np.abs(comov_dist[1] - comov_dist[0]) + ) integrand[index] = fourier_profile * mean_density * cddf_NHI[index] @@ -183,8 +223,9 @@ def main(): if not args.positive_fvoigt: Fvoigt = -Fvoigt - np.savetxt(args.output, np.c_[k, Fvoigt], header='k[hMpc^-1] FVoigt') + np.savetxt(args.output, np.c_[k, Fvoigt], header="k[hMpc^-1] FVoigt") -if __name__ == '__main__': - main() +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_compute_pk_pksb.py b/py/picca/bin/picca_compute_pk_pksb.py old mode 100644 new mode 100755 index fbde082a7..90c277df6 --- a/py/picca/bin/picca_compute_pk_pksb.py +++ b/py/picca/bin/picca_compute_pk_pksb.py @@ -1,50 +1,78 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -import sys -import os import argparse +import os +import sys + +import camb import fitsio -import numpy as np -from scipy.optimize import curve_fit import matplotlib.pyplot as plt -import camb import nbodykit.cosmology.correlation +import numpy as np +from scipy.optimize import curve_fit -from picca.utils import userprint from picca.constants import SPEED_LIGHT +from picca.utils import userprint -def main(cmdargs): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('-i','--ini', type=str, required=True, - help='Input config file for CAMB') - - parser.add_argument('-o','--out', type=str, required=True, - help='Output FITS file') - - parser.add_argument('--H0', type=float, required=False, default=None, - help='Hubble parameter, if not given use the one from the config file') - - parser.add_argument('--fid-Ok', type=float, default=None, required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') - - parser.add_argument('--fid-wl', type=float, default=None, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') - - parser.add_argument('--z-ref', type=float, required=False, default=None, - help='Power-spectrum redshift, if not given use the one from the config file') - parser.add_argument('--plot', action='store_true', required=False, - help='Plot the resulting correlation functions and power-spectra') +def main(cmdargs): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "-i", "--ini", type=str, required=True, help="Input config file for CAMB" + ) + + parser.add_argument("-o", "--out", type=str, required=True, help="Output FITS file") + + parser.add_argument( + "--H0", + type=float, + required=False, + default=None, + help="Hubble parameter, if not given use the one from the config file", + ) + + parser.add_argument( + "--fid-Ok", + type=float, + default=None, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) + + parser.add_argument( + "--fid-wl", + type=float, + default=None, + required=False, + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) + + parser.add_argument( + "--z-ref", + type=float, + required=False, + default=None, + help="Power-spectrum redshift, if not given use the one from the config file", + ) + + parser.add_argument( + "--plot", + action="store_true", + required=False, + help="Plot the resulting correlation functions and power-spectra", + ) args = parser.parse_args(cmdargs) ### Parameters kmin and kmax to get exactly same as DR12 - minkh = 1.e-4 + minkh = 1.0e-4 maxkh = 1.1525e3 npoints = 814 - userprint('INFO: running CAMB on {}'.format(args.ini)) + userprint("INFO: running CAMB on {}".format(args.ini)) pars = camb.read_ini(os.path.expandvars(args.ini)) pars.Transfer.kmax = maxkh if not args.z_ref is None: @@ -57,118 +85,120 @@ def main(cmdargs): pars.DarkEnergy.w = args.fid_wl results = camb.get_results(pars) - k, z, pk = results.get_matter_power_spectrum(minkh=minkh, maxkh=pars.Transfer.kmax, npoints=npoints) + k, z, pk = results.get_matter_power_spectrum( + minkh=minkh, maxkh=pars.Transfer.kmax, npoints=npoints + ) pk = pk[1] pars = results.Params pars2 = results.get_derived_params() ### Save the parameters cat = {} - cat['H0'] = pars.H0 - cat['ombh2'] = pars.ombh2 - cat['omch2'] = pars.omch2 - cat['omnuh2'] = pars.omnuh2 - cat['OK'] = pars.omk - cat['OL'] = results.get_Omega('de') - cat['ORPHOTON'] = results.get_Omega('photon') - cat['ORNEUTRI'] = results.get_Omega('neutrino') - cat['OR'] = cat['ORPHOTON']+cat['ORNEUTRI'] - cat['OM'] = (cat['ombh2']+cat['omch2']+cat['omnuh2'])/(cat['H0']/100.)**2 - cat['W'] = pars.DarkEnergy.w - cat['TCMB'] = pars.TCMB - cat['NS'] = pars.InitPower.ns - cat['ZREF'] = pars.Transfer.PK_redshifts[0] - cat['SIGMA8_ZREF'] = results.get_sigma8()[0] - cat['SIGMA8_Z0'] = results.get_sigma8()[1] - cat['F_ZREF'] = results.get_fsigma8()[0]/results.get_sigma8()[0] - cat['F_Z0'] = results.get_fsigma8()[1]/results.get_sigma8()[1] - cat['ZDRAG'] = pars2['zdrag'] - cat['RDRAG'] = pars2['rdrag'] - - c = SPEED_LIGHT/1000. ## km/s - h = cat['H0']/100. - dh = c/(results.hubble_parameter(cat['ZREF'])/h) - dm = (1.+cat['ZREF'])*results.angular_diameter_distance(cat['ZREF'])*h - cat['DH'] = dh - cat['DM'] = dm - cat['DHoRD'] = cat['DH']/(cat['RDRAG']*h) - cat['DMoRD'] = cat['DM']/(cat['RDRAG']*h) + cat["H0"] = pars.H0 + cat["ombh2"] = pars.ombh2 + cat["omch2"] = pars.omch2 + cat["omnuh2"] = pars.omnuh2 + cat["OK"] = pars.omk + cat["OL"] = results.get_Omega("de") + cat["ORPHOTON"] = results.get_Omega("photon") + cat["ORNEUTRI"] = results.get_Omega("neutrino") + cat["OR"] = cat["ORPHOTON"] + cat["ORNEUTRI"] + cat["OM"] = (cat["ombh2"] + cat["omch2"] + cat["omnuh2"]) / (cat["H0"] / 100.0) ** 2 + cat["W"] = pars.DarkEnergy.w + cat["TCMB"] = pars.TCMB + cat["NS"] = pars.InitPower.ns + cat["ZREF"] = pars.Transfer.PK_redshifts[0] + cat["SIGMA8_ZREF"] = results.get_sigma8()[0] + cat["SIGMA8_Z0"] = results.get_sigma8()[1] + cat["F_ZREF"] = results.get_fsigma8()[0] / results.get_sigma8()[0] + cat["F_Z0"] = results.get_fsigma8()[1] / results.get_sigma8()[1] + cat["ZDRAG"] = pars2["zdrag"] + cat["RDRAG"] = pars2["rdrag"] + + c = SPEED_LIGHT / 1000.0 ## km/s + h = cat["H0"] / 100.0 + dh = c / (results.hubble_parameter(cat["ZREF"]) / h) + dm = (1.0 + cat["ZREF"]) * results.angular_diameter_distance(cat["ZREF"]) * h + cat["DH"] = dh + cat["DM"] = dm + cat["DHoRD"] = cat["DH"] / (cat["RDRAG"] * h) + cat["DMoRD"] = cat["DM"] / (cat["RDRAG"] * h) ### Get the Side-Bands ### Follow 2.2.1 of Kirkby et al. 2013: https://arxiv.org/pdf/1301.3456.pdf - coef_Planck2015 = (cat['H0']/67.31)*(cat['RDRAG']/147.334271564563) - sb1_rmin = 50.*coef_Planck2015 - sb1_rmax = 82.*coef_Planck2015 - sb2_rmin = 150.*coef_Planck2015 - sb2_rmax = 190.*coef_Planck2015 - xi = nbodykit.cosmology.correlation.pk_to_xi(k,pk) - r = np.logspace(-7., 3.5, 10000) + coef_Planck2015 = (cat["H0"] / 67.31) * (cat["RDRAG"] / 147.334271564563) + sb1_rmin = 50.0 * coef_Planck2015 + sb1_rmax = 82.0 * coef_Planck2015 + sb2_rmin = 150.0 * coef_Planck2015 + sb2_rmax = 190.0 * coef_Planck2015 + xi = nbodykit.cosmology.correlation.pk_to_xi(k, pk) + r = np.logspace(-7.0, 3.5, 10000) xi = xi(r) - def f_xiSB(r,am3,am2,am1,a0,a1): - par = [am3,am2,am1,a0,a1] - model = np.zeros((len(par),r.size)) - tw = r!=0. - model[0,tw] = par[0]/r[tw]**3 - model[1,tw] = par[1]/r[tw]**2 - model[2,tw] = par[2]/r[tw]**1 - model[3,tw] = par[3] - model[4,:] = par[4]*r + def f_xiSB(r, am3, am2, am1, a0, a1): + par = [am3, am2, am1, a0, a1] + model = np.zeros((len(par), r.size)) + tw = r != 0.0 + model[0, tw] = par[0] / r[tw] ** 3 + model[1, tw] = par[1] / r[tw] ** 2 + model[2, tw] = par[2] / r[tw] ** 1 + model[3, tw] = par[3] + model[4, :] = par[4] * r model = np.array(model) return model.sum(axis=0) - w = ((r>=sb1_rmin) & (r=sb2_rmin) & (r=sb1_rmin-2.) & (r=sb2_rmax-2.) & (r= sb1_rmin) & (r < sb1_rmax)) | ((r >= sb2_rmin) & (r < sb2_rmax)) + sigma = 0.1 * np.ones(xi.size) + sigma[(r >= sb1_rmin - 2.0) & (r < sb1_rmin + 2.0)] = 1.0e-6 + sigma[(r >= sb2_rmax - 2.0) & (r < sb2_rmax + 2.0)] = 1.0e-6 popt, pcov = curve_fit(f_xiSB, r[w], xi[w], sigma=sigma[w]) model = f_xiSB(r, *popt) xiSB = xi.copy() - ww = (r>=sb1_rmin) & (r= sb1_rmin) & (r < sb2_rmax) xiSB[ww] = model[ww] - pkSB = nbodykit.cosmology.correlation.xi_to_pk(r,xiSB,extrap=True) + pkSB = nbodykit.cosmology.correlation.xi_to_pk(r, xiSB, extrap=True) pkSB = pkSB(k) - pkSB *= pk[-1]/pkSB[-1] + pkSB *= pk[-1] / pkSB[-1] - out = fitsio.FITS(args.out,'rw',clobber=True) - head = [{'name':k,'value':float(v)} for k,v in cat.items() ] - out.write([k,pk,pkSB],names=['K','PK','PKSB'],header=head,extname='PK') + out = fitsio.FITS(args.out, "rw", clobber=True) + head = [{"name": k, "value": float(v)} for k, v in cat.items()] + out.write([k, pk, pkSB], names=["K", "PK", "PKSB"], header=head, extname="PK") out.close() if args.plot: - plt.plot(r,xi*r**2,label='Full') - w = (r>=sb1_rmin) & (r=sb2_rmin) & (r= sb1_rmin) & (r < sb1_rmax) + plt.plot(r[w], xi[w] * r[w] ** 2, label="SB1") + w = (r >= sb2_rmin) & (r < sb2_rmax) + plt.plot(r[w], xi[w] * r[w] ** 2, label="SB2") + plt.plot(r, xiSB * r**2, label="noBAO") + plt.xlabel(r"$r\,[h^{-1}\,\mathrm{Mpc}]$") + plt.ylabel(r"$r^{2}\,\xi(r)$") plt.legend() plt.grid() plt.show() - plt.plot(k,pk,label='Full') - plt.plot(k,pkSB,label='noBAO') - plt.xscale('log') - plt.yscale('log') - plt.xlabel(r'$k$') - plt.ylabel(r'$P(k)$') + plt.plot(k, pk, label="Full") + plt.plot(k, pkSB, label="noBAO") + plt.xscale("log") + plt.yscale("log") + plt.xlabel(r"$k$") + plt.ylabel(r"$P(k)$") plt.legend() plt.grid() plt.show() - plt.plot(k,pk-pkSB,label='BAO') - plt.xscale('log') - plt.xlabel(r'$k$') - plt.ylabel(r'$P(k)$') + plt.plot(k, pk - pkSB, label="BAO") + plt.xscale("log") + plt.xlabel(r"$k$") + plt.ylabel(r"$P(k)$") plt.legend() plt.grid() plt.show() -if __name__ == '__main__': - cmdargs=sys.argv[1:] - main(cmdargs) \ No newline at end of file +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_convert_transmission.py b/py/picca/bin/picca_convert_transmission.py old mode 100644 new mode 100755 index 1dc90fa23..b13f00ca6 --- a/py/picca/bin/picca_convert_transmission.py +++ b/py/picca/bin/picca_convert_transmission.py @@ -1,70 +1,152 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import argparse +import sys from picca import raw_io -if __name__ == '__main__': - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Script to convert noiseless transmission files to delta picca files') - - parser.add_argument('--object-cat', type=str, default=None, required=True, - help='Path to a catalog of objects to get the transmission from') - - parser.add_argument('--in-dir', type=str, default=None, required=False, - help='Desi formated data directory to transmission files') - - parser.add_argument('--in-files', type=str, default=None, required=False, - help='List of transmission files.', nargs='*') - - parser.add_argument('--out-dir', type=str, default=None, required=True, - help='Output directory') - - parser.add_argument('--lambda-min', type=float, default=3600., required=False, - help='Lower limit on observed wavelength [Angstrom]') - - parser.add_argument('--lambda-max', type=float, default=5500., required=False, - help='Upper limit on observed wavelength [Angstrom]') - - parser.add_argument('--lambda-rest-min', type=float, default=1040., required=False, - help='Lower limit on rest frame wavelength [Angstrom]') - - parser.add_argument('--lambda-rest-max', type=float, default=1200., required=False, - help='Upper limit on rest frame wavelength [Angstrom]') - - parser.add_argument('--delta-log-lambda', type=float, default=3.e-4, required=False, - help='Size of the rebined pixels in log lambda') - - parser.add_argument('--delta-lambda', type=float, default=None, required=False, - help='Size of the rebined pixels in log lambda') - - parser.add_argument('--linear-spacing', action="store_true", default=False, required=False, - help='Whether to use linear bins in lambda.') - - parser.add_argument('--nspec', type=int, default=None, required=False, - help="Number of spectra to fit, if None then run on all files") - - parser.add_argument('--use-old-weights', action="store_true", default=False, required=False, - help='Whether to use the old weighting scheme for raw deltas.') - - parser.add_argument('--tracer',type=str, default='F_LYA', required=False, - help='Tracer to use') - - parser.add_argument('--use-splines',action="store_true", default=False, required=False, - help='Use splines to compute mean flux and variance') - - args = parser.parse_args() - - raw_io.convert_transmission_to_deltas(args.object_cat, args.out_dir, in_dir=args.in_dir, - in_filenames=args.in_files,tracer=args.tracer, - lambda_min=args.lambda_min, - lambda_max=args.lambda_max, - lambda_min_rest_frame=args.lambda_rest_min, - lambda_max_rest_frame=args.lambda_rest_max, - delta_log_lambda=args.delta_log_lambda, - delta_lambda=args.delta_lambda, - lin_spaced=args.linear_spacing, - max_num_spec=args.nspec, - use_old_weights=args.use_old_weights, - use_splines=args.use_splines) +def main(cmdargs): + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Script to convert noiseless transmission files to delta picca files", + ) + + parser.add_argument( + "--object-cat", + type=str, + default=None, + required=True, + help="Path to a catalog of objects to get the transmission from", + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=False, + help="Desi formated data directory to transmission files", + ) + + parser.add_argument( + "--in-files", + type=str, + default=None, + required=False, + help="List of transmission files.", + nargs="*", + ) + + parser.add_argument( + "--out-dir", type=str, default=None, required=True, help="Output directory" + ) + + parser.add_argument( + "--lambda-min", + type=float, + default=3600.0, + required=False, + help="Lower limit on observed wavelength [Angstrom]", + ) + + parser.add_argument( + "--lambda-max", + type=float, + default=5500.0, + required=False, + help="Upper limit on observed wavelength [Angstrom]", + ) + + parser.add_argument( + "--lambda-rest-min", + type=float, + default=1040.0, + required=False, + help="Lower limit on rest frame wavelength [Angstrom]", + ) + + parser.add_argument( + "--lambda-rest-max", + type=float, + default=1200.0, + required=False, + help="Upper limit on rest frame wavelength [Angstrom]", + ) + + parser.add_argument( + "--delta-log-lambda", + type=float, + default=3.0e-4, + required=False, + help="Size of the rebined pixels in log lambda", + ) + + parser.add_argument( + "--delta-lambda", + type=float, + default=None, + required=False, + help="Size of the rebined pixels in log lambda", + ) + + parser.add_argument( + "--linear-spacing", + action="store_true", + default=False, + required=False, + help="Whether to use linear bins in lambda.", + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, + required=False, + help="Number of spectra to fit, if None then run on all files", + ) + + parser.add_argument( + "--use-old-weights", + action="store_true", + default=False, + required=False, + help="Whether to use the old weighting scheme for raw deltas.", + ) + + parser.add_argument( + "--tracer", type=str, default="F_LYA", required=False, help="Tracer to use" + ) + + parser.add_argument( + "--use-splines", + action="store_true", + default=False, + required=False, + help="Use splines to compute mean flux and variance", + ) + + args = parser.parse_args(cmdargs) + + raw_io.convert_transmission_to_deltas( + args.object_cat, + args.out_dir, + in_dir=args.in_dir, + in_filenames=args.in_files, + tracer=args.tracer, + lambda_min=args.lambda_min, + lambda_max=args.lambda_max, + lambda_min_rest_frame=args.lambda_rest_min, + lambda_max_rest_frame=args.lambda_rest_max, + delta_log_lambda=args.delta_log_lambda, + delta_lambda=args.delta_lambda, + lin_spaced=args.linear_spacing, + max_num_spec=args.nspec, + use_old_weights=args.use_old_weights, + use_splines=args.use_splines, + ) + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_delta_extraction.py b/py/picca/bin/picca_delta_extraction.py old mode 100644 new mode 100755 index 509cb2e6a..5b1f69777 --- a/py/picca/bin/picca_delta_extraction.py +++ b/py/picca/bin/picca_delta_extraction.py @@ -5,9 +5,10 @@ spectra for the specified absorption line. Follow the procedure described in section 2.4 of du Mas des Bourboux et al. 2020 (In prep). """ +import argparse import logging +import sys import time -import argparse from picca.delta_extraction.survey import Survey @@ -18,18 +19,19 @@ def main(cmdargs=None): """Compute delta field""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the delta field ' - 'from a list of spectra')) + description=("Compute the delta field " "from a list of spectra"), + ) parser.add_argument( - 'config_file', + "config_file", type=str, default=None, - help= - ('Configuration file. To learn about all the available options ' - 'check the configuration tutorial in ' - '$PICCA/tutorials/delta_extraction/picca_delta_extraction_tutorial.ipynb' - )) + help=( + "Configuration file. To learn about all the available options " + "check the configuration tutorial in " + "$PICCA/tutorials/delta_extraction/picca_delta_extraction_tutorial.ipynb" + ), + ) args = parser.parse_args(cmdargs) @@ -69,3 +71,8 @@ def main(cmdargs=None): t1 = time.time() module_logger.info(f"Total time ellapsed: {t1-t0}") module_logger.info("Done") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_dmat.py b/py/picca/bin/picca_dmat.py old mode 100644 new mode 100755 index 61bce4070..0a80c313f --- a/py/picca/bin/picca_dmat.py +++ b/py/picca/bin/picca_dmat.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Computes the distortion matrix between two delta fields. This module follow the procedure described in sections 3.5 of du Mas des @@ -6,8 +6,9 @@ """ import argparse import multiprocessing +import sys import time -from multiprocessing import Lock, Pool, Value, cpu_count +from multiprocessing import Lock, Value, cpu_count import fitsio import numpy as np @@ -99,10 +100,14 @@ def main(cmdargs=None): "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" ) - parser.add_argument('--rmu-binning', action="store_true", - help=('Estimate in r,mu binning. np becomes mu bins.' - ' nt becomes r bins. rp min max is always 0, 1') - ) + parser.add_argument( + "--rmu-binning", + action="store_true", + help=( + "Estimate in r,mu binning. np becomes mu bins." + " nt becomes r bins. rp min max is always 0, 1" + ), + ) parser.add_argument( "--coef-binning-model", @@ -571,36 +576,38 @@ def main(cmdargs=None): "value": args.fid_Or, "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", }, + {"name": "NPUSED", "value": num_pairs_used, "comment": "Number of used pairs"}, { - 'name': 'NPUSED', - 'value': num_pairs_used, - 'comment': 'Number of used pairs' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - }, { - 'name': "RMU_BIN", - 'value': cf.rmu_binning, - 'comment': 'True if binned in r, mu' - } - ] + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + { + "name": "RMU_BIN", + "value": cf.rmu_binning, + "comment": "True if binned in r, mu", + }, + ] dmat_name = "DM" if blinding != "none": dmat_name += "_BLIND" @@ -623,3 +630,8 @@ def main(cmdargs=None): t3 = time.time() userprint(f"picca_dmat.py - Time total : {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_export.py b/py/picca/bin/picca_export.py old mode 100644 new mode 100755 index 886b7ab16..5c54ed247 --- a/py/picca/bin/picca_export.py +++ b/py/picca/bin/picca_export.py @@ -1,21 +1,25 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Export auto and cross-correlation for the fitter.""" import argparse +import os.path +import sys + import fitsio +import h5py import numpy as np import scipy.interpolate import scipy.linalg -import h5py -import os.path -import sys - from numpy.polynomial.legendre import legvander from scipy.sparse import coo_array from picca.utils import ( - smooth_cov, compute_cov, compute_cov_boot, - calculate_xi_ell, get_legendre_bins) -from picca.utils import userprint + calculate_xi_ell, + compute_cov, + compute_cov_boot, + get_legendre_bins, + smooth_cov, + userprint, +) # TODO: add tags here when we are allowed to unblind them UNBLINDABLE_STRATEGIES = ["none", "desi_m2", "desi_y1", "desi_y3"] @@ -25,118 +29,143 @@ def main(cmdargs=None): """Export auto and cross-correlation for the fitter.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Export auto and cross-correlation for the fitter.') + description="Export auto and cross-correlation for the fitter.", + ) parser.add_argument( - '--data', + "--data", type=str, default=None, required=True, - help='Correlation produced via picca_cf.py, picca_xcf.py, ...') + help="Correlation produced via picca_cf.py, picca_xcf.py, ...", + ) parser.add_argument( - '--out', - type=str, - default=None, - required=True, - help='Output file name') + "--out", type=str, default=None, required=True, help="Output file name" + ) parser.add_argument( - '--dmat', + "--dmat", type=str, default=None, required=False, - help=('Distortion matrix produced via picca_dmat.py, picca_xdmat.py... ' - '(if not provided will be identity)')) - - parser.add_argument('--multipoles', action="store_true", - help='Use multipole extension') - parser.add_argument('--lmax-model', type=int, default=6, - help=('Max multipole ell for the model if rmu-binning' - '. Note odd multipoles are removed from auto.') - ) - parser.add_argument('--lmax-data', type=int, default=10, - help=('Max multipole ell for the output if rmu-binning' - '. Note odd multipoles are removed from auto.') - ) + help=( + "Distortion matrix produced via picca_dmat.py, picca_xdmat.py... " + "(if not provided will be identity)" + ), + ) + + parser.add_argument( + "--multipoles", action="store_true", help="Use multipole extension" + ) + parser.add_argument( + "--lmax-model", + type=int, + default=6, + help=( + "Max multipole ell for the model if rmu-binning" + ". Note odd multipoles are removed from auto." + ), + ) + parser.add_argument( + "--lmax-data", + type=int, + default=10, + help=( + "Max multipole ell for the output if rmu-binning" + ". Note odd multipoles are removed from auto." + ), + ) parser.add_argument( - '--cov', + "--cov", type=str, default=None, required=False, - help=('Covariance matrix (if not provided will be calculated by ' - 'subsampling)')) + help=( + "Covariance matrix (if not provided will be calculated by " "subsampling)" + ), + ) parser.add_argument( - '--cor', + "--cor", type=str, default=None, required=False, - help=('Correlation matrix (if not provided will be calculated by ' - 'subsampling)')) + help=( + "Correlation matrix (if not provided will be calculated by " "subsampling)" + ), + ) parser.add_argument( - '--remove-shuffled-correlation', + "--remove-shuffled-correlation", type=str, default=None, required=False, - help='Remove a correlation from shuffling the distribution of los') + help="Remove a correlation from shuffling the distribution of los", + ) parser.add_argument( - '--num-boot-cov', - type=int, default=0, - help='Number of bootstrap realizations. <=0 turns it off.') + "--num-boot-cov", + type=int, + default=0, + help="Number of bootstrap realizations. <=0 turns it off.", + ) parser.add_argument( - '--do-not-smooth-cov', - action='store_true', + "--do-not-smooth-cov", + action="store_true", default=False, - help='Do not smooth the covariance matrix') + help="Do not smooth the covariance matrix", + ) parser.add_argument( - '--smooth-per-r-par', - action='store_true', + "--smooth-per-r-par", + action="store_true", default=False, - help='Consider different correlation coefficients per r_par') + help="Consider different correlation coefficients per r_par", + ) parser.add_argument( - '--blind-corr-type', + "--blind-corr-type", default=None, - choices=['lyaxlya', 'lyaxlyb', 'qsoxlya', 'qsoxlyb'], - help='Type of correlation. Required to apply blinding in DESI') + choices=["lyaxlya", "lyaxlyb", "qsoxlya", "qsoxlyb"], + help="Type of correlation. Required to apply blinding in DESI", + ) args = parser.parse_args(cmdargs) hdul = fitsio.FITS(args.data) - r_par = np.array(hdul[1]['RP'][:]) - r_trans = np.array(hdul[1]['RT'][:]) - z = np.array(hdul[1]['Z'][:]) - num_pairs = np.array(hdul[1]['NB'][:]) - weights = np.array(hdul[2]['WE'][:]) + r_par = np.array(hdul[1]["RP"][:]) + r_trans = np.array(hdul[1]["RT"][:]) + z = np.array(hdul[1]["Z"][:]) + num_pairs = np.array(hdul[1]["NB"][:]) + weights = np.array(hdul[2]["WE"][:]) - if 'DA_BLIND' in hdul[2].get_colnames(): - xi = np.array(hdul[2]['DA_BLIND'][:]) - data_name = 'DA_BLIND' + if "DA_BLIND" in hdul[2].get_colnames(): + xi = np.array(hdul[2]["DA_BLIND"][:]) + data_name = "DA_BLIND" else: - xi = np.array(hdul[2]['DA'][:]) - data_name = 'DA' + xi = np.array(hdul[2]["DA"][:]) + data_name = "DA" head = hdul[1].read_header() - num_bins_r_par = head['NP'] - num_bins_r_trans = head['NT'] - r_trans_max = head['RTMAX'] - r_par_min = head['RPMIN'] - r_par_max = head['RPMAX'] + num_bins_r_par = head["NP"] + num_bins_r_trans = head["NT"] + r_trans_max = head["RTMAX"] + r_par_min = head["RPMIN"] + r_par_max = head["RPMAX"] nsamples = xi.shape[0] is_x_correlation = r_par_min < 0 if "BLINDING" in head: blinding = head["BLINDING"] - if blinding == 'minimal': - blinding = 'corr_yshift' - userprint("The minimal strategy is no longer supported." - "Automatically switch to corr_yshift.") + if blinding == "minimal": + blinding = "corr_yshift" + userprint( + "The minimal strategy is no longer supported." + "Automatically switch to corr_yshift." + ) else: # if BLINDING keyword not present (old file), ignore blinding blinding = "none" @@ -144,11 +173,11 @@ def main(cmdargs=None): if args.remove_shuffled_correlation is not None: hdul = fitsio.FITS(args.remove_shuffled_correlation) - xi_shuffled = hdul['COR'][data_name][:] - weight_shuffled = hdul['COR']['WE'][:] + xi_shuffled = hdul["COR"][data_name][:] + weight_shuffled = hdul["COR"]["WE"][:] xi_shuffled = (xi_shuffled * weight_shuffled).sum(axis=1) weight_shuffled = weight_shuffled.sum(axis=1) - w = weight_shuffled > 0. + w = weight_shuffled > 0.0 xi_shuffled[w] /= weight_shuffled[w] hdul.close() xi -= xi_shuffled[:, None] @@ -156,7 +185,7 @@ def main(cmdargs=None): if args.multipoles: userprint("Smoothing is turned off for multipoles") args.do_not_smooth_cov = True - assert head['RMU_BIN'] + assert head["RMU_BIN"] ells_out = np.arange(args.lmax_data + 1) if not is_x_correlation: @@ -181,39 +210,56 @@ def main(cmdargs=None): r_par = np.repeat(ells_out, num_bins_r_trans) xi, weights = calculate_xi_ell( - ells_out, xi, weights, num_bins_r_par, is_x_correlation) + ells_out, xi, weights, num_bins_r_par, is_x_correlation + ) header_multipole = [ - {'name': 'MLTPOLE', - 'value': True, - 'comment': 'Specifying if in multipoles.'}, - {'name': 'NL_DATA', - 'value': nell_out, - 'comment': 'Number of output multipoles'} + { + "name": "MLTPOLE", + "value": True, + "comment": "Specifying if in multipoles.", + }, + { + "name": "NL_DATA", + "value": nell_out, + "comment": "Number of output multipoles", + }, ] else: ells_out = None header_multipole = [ - {'name': 'MLTPOLE', - 'value': False, - 'comment': 'Specifying if in multipoles.'}] + { + "name": "MLTPOLE", + "value": False, + "comment": "Specifying if in multipoles.", + } + ] if args.cov is not None: - userprint(("INFO: The covariance-matrix will be read from file: " - "{}").format(args.cov)) + userprint( + ("INFO: The covariance-matrix will be read from file: " "{}").format( + args.cov + ) + ) hdul = fitsio.FITS(args.cov) - covariance = hdul[1]['CO'][:] + covariance = hdul[1]["CO"][:] hdul.close() elif args.cor is not None: - userprint(("INFO: The correlation-matrix will be read from file: " - "{}").format(args.cor)) + userprint( + ("INFO: The correlation-matrix will be read from file: " "{}").format( + args.cor + ) + ) hdul = fitsio.FITS(args.cor) - correlation = hdul[1]['CO'][:] + correlation = hdul[1]["CO"][:] hdul.close() - if ((correlation.min() < -1.) or (correlation.min() > 1.) or - (correlation.max() < -1.) or (correlation.max() > 1.) or - np.any(np.diag(correlation) != 1.)): - userprint(("WARNING: The correlation-matrix has some incorrect " - "values")) + if ( + (correlation.min() < -1.0) + or (correlation.min() > 1.0) + or (correlation.max() < -1.0) + or (correlation.max() > 1.0) + or np.any(np.diag(correlation) != 1.0) + ): + userprint(("WARNING: The correlation-matrix has some incorrect " "values")) var = np.diagonal(correlation) correlation = correlation / np.sqrt(var * var[:, None]) covariance = compute_cov(xi, weights) @@ -221,10 +267,12 @@ def main(cmdargs=None): covariance = correlation * np.sqrt(var * var[:, None]) else: delta_r_par = (r_par_max - r_par_min) / num_bins_r_par - delta_r_trans = (r_trans_max - 0.) / num_bins_r_trans + delta_r_trans = (r_trans_max - 0.0) / num_bins_r_trans if args.num_boot_cov > 0: - userprint(f"INFO: Covariance with {args.num_boot_cov} bootstrap realizations.") + userprint( + f"INFO: Covariance with {args.num_boot_cov} bootstrap realizations." + ) covariance = compute_cov_boot(xi, weights, nboots=args.num_boot_cov) else: covariance = compute_cov(xi, weights) @@ -233,14 +281,18 @@ def main(cmdargs=None): userprint("INFO: The covariance will not be smoothed") else: userprint("INFO: The covariance will be smoothed") - if args.smooth_per_r_par : + if args.smooth_per_r_par: userprint("INFO: with different correlation coefficients per r_par") covariance = smooth_cov( - None, None, r_par, r_trans, + None, + None, + r_par, + r_trans, delta_r_trans=delta_r_trans, delta_r_par=delta_r_par, covariance=covariance, - per_r_par=args.smooth_per_r_par) + per_r_par=args.smooth_per_r_par, + ) xi = (xi * weights).sum(axis=0) weights = weights.sum(axis=0) @@ -256,25 +308,29 @@ def main(cmdargs=None): hdul = fitsio.FITS(args.dmat) head_dmat = hdul[1].read_header() - if data_name == "DA_BLIND" and 'DM_BLIND' in hdul[1].get_colnames(): - dmat = np.array(hdul[1]['DM_BLIND'][:]) - dmat_name = 'DM_BLIND' + if data_name == "DA_BLIND" and "DM_BLIND" in hdul[1].get_colnames(): + dmat = np.array(hdul[1]["DM_BLIND"][:]) + dmat_name = "DM_BLIND" elif data_name == "DA_BlIND": - userprint("Blinded correlations were given but distortion matrix " - "is unblinded. These files should not mix. Exiting...") + userprint( + "Blinded correlations were given but distortion matrix " + "is unblinded. These files should not mix. Exiting..." + ) sys.exit(1) - elif 'DM_BLIND' in hdul[1].get_colnames(): - userprint("Non-blinded correlations were given but distortion matrix " - "is blinded. These files should not mix. Exiting...") + elif "DM_BLIND" in hdul[1].get_colnames(): + userprint( + "Non-blinded correlations were given but distortion matrix " + "is blinded. These files should not mix. Exiting..." + ) sys.exit(1) else: - dmat = hdul[1]['DM'][:] - dmat_name = 'DM' + dmat = hdul[1]["DM"][:] + dmat_name = "DM" try: - r_par_dmat = hdul[2]['RP'][:] - r_trans_dmat = hdul[2]['RT'][:] - z_dmat = hdul[2]['Z'][:] + r_par_dmat = hdul[2]["RP"][:] + r_trans_dmat = hdul[2]["RT"][:] + z_dmat = hdul[2]["Z"][:] except IOError: r_par_dmat = r_par.copy() r_trans_dmat = r_trans.copy() @@ -286,31 +342,32 @@ def main(cmdargs=None): hdul.close() if args.multipoles: - assert head_dmat['RMU_BIN'] - nmu_dmat = head_dmat['NP'] * head_dmat['COEFMOD'] - nr_dmat = head_dmat['NT'] * head_dmat['COEFMOD'] - dr_dmat = head_dmat['RTMAX'] / nr_dmat + assert head_dmat["RMU_BIN"] + nmu_dmat = head_dmat["NP"] * head_dmat["COEFMOD"] + nr_dmat = head_dmat["NT"] * head_dmat["COEFMOD"] + dr_dmat = head_dmat["RTMAX"] / nr_dmat ells_model = np.arange(args.lmax_model + 1) if not is_x_correlation: ells_model = ells_model[ells_model % 2 == 0] nell_model = ells_model.size # From model multipoles to transverse-radial interpolation matrix. - ell_to_tr_matrix = legvander( - r_par_dmat, args.lmax_model)[:, ells_model] + ell_to_tr_matrix = legvander(r_par_dmat, args.lmax_model)[:, ells_model] cols = np.floor(r_trans_dmat / dr_dmat).astype(int) cols = np.repeat(cols, nell_model) + np.tile( - np.arange(nell_model) * nr_dmat, cols.size) + np.arange(nell_model) * nr_dmat, cols.size + ) rows = np.repeat(np.arange(dmat.shape[1]), nell_model) ell_to_tr_matrix = coo_array( (ell_to_tr_matrix.ravel(), (rows, cols)), - shape=(dmat.shape[1], nell_model * nr_dmat)) + shape=(dmat.shape[1], nell_model * nr_dmat), + ) r_par_dmat = np.repeat(ells_model, nr_dmat) r_trans_dmat = np.tile( - r_trans_dmat.reshape(nmu_dmat, nr_dmat).mean(0), nell_model) - z_dmat = np.tile( - z_dmat.reshape(nmu_dmat, nr_dmat).mean(0), nell_model) + r_trans_dmat.reshape(nmu_dmat, nr_dmat).mean(0), nell_model + ) + z_dmat = np.tile(z_dmat.reshape(nmu_dmat, nr_dmat).mean(0), nell_model) dmat = ell_to_tr_matrix.T.dot(dmat.T).T del ell_to_tr_matrix, rows, cols @@ -319,8 +376,7 @@ def main(cmdargs=None): assert xi.size == nell_out * num_bins_r_trans tr_to_ell_matrix = np.zeros((xi.size, dmat.shape[0])) - leg_ells = get_legendre_bins( - ells_out, num_bins_r_par, is_x_correlation) + leg_ells = get_legendre_bins(ells_out, num_bins_r_par, is_x_correlation) for i in range(xi.size): ell = i // num_bins_r_trans @@ -330,118 +386,133 @@ def main(cmdargs=None): dmat = tr_to_ell_matrix.dot(dmat) del tr_to_ell_matrix - header_multipole.append({ - 'name': 'NL_MODEL', - 'value': nell_model, - 'comment': 'Number of model multipoles'}) + header_multipole.append( + { + "name": "NL_MODEL", + "value": nell_model, + "comment": "Number of model multipoles", + } + ) else: dmat = np.eye(len(xi)) r_par_dmat = r_par.copy() r_trans_dmat = r_trans.copy() z_dmat = z.copy() - dmat_name = 'DM_EMPTY' + dmat_name = "DM_EMPTY" - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = [ - { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - }, { - 'name': 'RPMIN', - 'value': r_par_min, - 'comment': 'Minimum r-parallel' - }, { - 'name': 'RPMAX', - 'value': r_par_max, - 'comment': 'Maximum r-parallel' - }, { - 'name': 'RTMAX', - 'value': r_trans_max, - 'comment': 'Maximum r-transverse' - }, { - 'name': 'NP', - 'value': num_bins_r_par, - 'comment': 'Number of bins in r-parallel' - }, { - 'name': 'NT', - 'value': num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' - }, { - 'name': 'OMEGAM', - 'value': head['OMEGAM'], - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': head['OMEGAR'], - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': head['OMEGAK'], - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': head['WL'], - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "RMU_BIN", - 'value': 'RMU_BIN' in head and head['RMU_BIN'] and not args.multipoles, - 'comment': 'True if binned in r, mu' - }, { - 'name': "NSAMPLES", 'value': nsamples, 'comment': 'Number of samples' - } + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + {"name": "RPMIN", "value": r_par_min, "comment": "Minimum r-parallel"}, + {"name": "RPMAX", "value": r_par_max, "comment": "Maximum r-parallel"}, + {"name": "RTMAX", "value": r_trans_max, "comment": "Maximum r-transverse"}, + { + "name": "NP", + "value": num_bins_r_par, + "comment": "Number of bins in r-parallel", + }, + { + "name": "NT", + "value": num_bins_r_trans, + "comment": "Number of bins in r-transverse", + }, + { + "name": "OMEGAM", + "value": head["OMEGAM"], + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": head["OMEGAR"], + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": head["OMEGAK"], + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": head["WL"], + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "RMU_BIN", + "value": "RMU_BIN" in head and head["RMU_BIN"] and not args.multipoles, + "comment": "True if binned in r, mu", + }, + {"name": "NSAMPLES", "value": nsamples, "comment": "Number of samples"}, ] + header_multipole comment = [ - 'R-parallel', 'R-transverse', 'Redshift', 'Correlation', - 'Covariance matrix', 'Distortion matrix', 'Number of pairs' + "R-parallel", + "R-transverse", + "Redshift", + "Correlation", + "Covariance matrix", + "Distortion matrix", + "Number of pairs", ] # Check if we need to unblind if blinding in UNBLINDABLE_STRATEGIES: userprint(f"'{blinding}' correlations are not blinded.") - blinding = 'none' - data_name = 'DA' - dmat_name = 'DM' + blinding = "none" + data_name = "DA" + dmat_name = "DM" # Check if we need blinding and apply it - if 'BLIND' in data_name or blinding != 'none': - blinding_dir = '/global/cfs/projectdirs/desi/science/lya/y1-kp6/blinding/' - blinding_templates = {'desi_y3': {'standard': 'y3_blinding_v3_standard_18_12_2022.h5', - 'grid': 'y3_blinding_v3_regular_grid_18_12_2022.h5'}} + if "BLIND" in data_name or blinding != "none": + blinding_dir = "/global/cfs/projectdirs/desi/science/lya/y1-kp6/blinding/" + blinding_templates = { + "desi_y3": { + "standard": "y3_blinding_v3_standard_18_12_2022.h5", + "grid": "y3_blinding_v3_regular_grid_18_12_2022.h5", + } + } if blinding in blinding_templates: userprint(f"Blinding using seed for {blinding}") else: - raise ValueError(f"Expected blinding to be one of {blinding_templates.keys()}." - f" Found {blinding}.") + raise ValueError( + f"Expected blinding to be one of {blinding_templates.keys()}." + f" Found {blinding}." + ) if args.blind_corr_type is None: raise ValueError("Blinding requires argument --blind_corr_type.") # Check type of correlation and get size and regular binning - if args.blind_corr_type in ['lyaxlya', 'lyaxlyb']: + if args.blind_corr_type in ["lyaxlya", "lyaxlyb"]: corr_size = 2500 - rp_interp_grid = np.arange(2., 202., 4) - rt_interp_grid = np.arange(2., 202., 4) - elif args.blind_corr_type in ['qsoxlya', 'qsoxlyb']: + rp_interp_grid = np.arange(2.0, 202.0, 4) + rt_interp_grid = np.arange(2.0, 202.0, 4) + elif args.blind_corr_type in ["qsoxlya", "qsoxlyb"]: corr_size = 5000 rp_interp_grid = np.arange(-197.99, 202.01, 4) - rt_interp_grid = np.arange(2., 202, 4) + rt_interp_grid = np.arange(2.0, 202, 4) else: - raise ValueError("Unknown correlation type: {}".format(args.blind_corr_type)) + raise ValueError( + "Unknown correlation type: {}".format(args.blind_corr_type) + ) if corr_size == num_bins_r_par * num_bins_r_trans: # Read the blinding file and get the right template - blinding_filename = blinding_dir + blinding_templates[blinding]['standard'] + blinding_filename = blinding_dir + blinding_templates[blinding]["standard"] else: # Read the regular grid blinding file and get the right template - blinding_filename = blinding_dir + blinding_templates[blinding]['grid'] + blinding_filename = blinding_dir + blinding_templates[blinding]["grid"] if not os.path.isfile(blinding_filename): - raise RuntimeError("Missing blinding file. Make sure you are running at" - " NERSC or contact picca developers") - blinding_file = h5py.File(blinding_filename, 'r') - hex_diff = np.array(blinding_file['blinding'][args.blind_corr_type]).astype(str) + raise RuntimeError( + "Missing blinding file. Make sure you are running at" + " NERSC or contact picca developers" + ) + blinding_file = h5py.File(blinding_filename, "r") + hex_diff = np.array(blinding_file["blinding"][args.blind_corr_type]).astype(str) diff_grid = np.array([float.fromhex(x) for x in hex_diff]) if corr_size == num_bins_r_par * num_bins_r_trans: @@ -449,31 +520,46 @@ def main(cmdargs=None): else: # Interpolate the blinding template on the regular grid interp = scipy.interpolate.RectBivariateSpline( - rp_interp_grid, rt_interp_grid, + rp_interp_grid, + rt_interp_grid, diff_grid.reshape(len(rp_interp_grid), len(rt_interp_grid)), - kx=3, ky=3) + kx=3, + ky=3, + ) diff = interp.ev(r_par, r_trans) if args.multipoles: diff = calculate_xi_ell( - ells_out, diff, [], num_bins_r_par, is_x_correlation)[0] + ells_out, diff, [], num_bins_r_par, is_x_correlation + )[0] # Check that the shapes match if np.shape(xi) != np.shape(diff): - raise RuntimeError("Unknown binning or wrong correlation type. Cannot blind." - " Please raise an issue or contact picca developers.") + raise RuntimeError( + "Unknown binning or wrong correlation type. Cannot blind." + " Please raise an issue or contact picca developers." + ) # Add blinding xi = xi + diff - results.write([xi, r_par, r_trans, z, covariance, dmat, num_pairs], - names=[data_name, 'RP', 'RT', 'Z', 'CO', dmat_name, 'NB'], - comment=comment, - header=header, - extname='COR') - comment = ['R-parallel model', 'R-transverse model', 'Redshift model'] - results.write([r_par_dmat, r_trans_dmat, z_dmat], - names=['DMRP', 'DMRT', 'DMZ'], - comment=comment, - extname='DMATTRI') + results.write( + [xi, r_par, r_trans, z, covariance, dmat, num_pairs], + names=[data_name, "RP", "RT", "Z", "CO", dmat_name, "NB"], + comment=comment, + header=header, + extname="COR", + ) + comment = ["R-parallel model", "R-transverse model", "Redshift model"] + results.write( + [r_par_dmat, r_trans_dmat, z_dmat], + names=["DMRP", "DMRT", "DMZ"], + comment=comment, + extname="DMATTRI", + ) results.close() + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_export_co.py b/py/picca/bin/picca_export_co.py old mode 100644 new mode 100755 index e8e2f739b..4b9f830fe --- a/py/picca/bin/picca_export_co.py +++ b/py/picca/bin/picca_export_co.py @@ -1,12 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Export auto and cross-correlation of catalog of objects for the fitter.""" -import sys import argparse +import sys + import fitsio import numpy as np import scipy.linalg -from picca.utils import smooth_cov, compute_cov, userprint +from picca.utils import compute_cov, smooth_cov, userprint def main(cmdargs): @@ -15,108 +16,135 @@ def main(cmdargs): fitter.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Export auto and cross-correlation of catalog of objects ' - 'for the fitter.')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--DD-file', - type=str, - default=None, - required=False, - help='File of the data x data auto-correlation') - - parser.add_argument('--RR-file', - type=str, - default=None, - required=False, - help='File of the random x random auto-correlation') - - parser.add_argument('--DR-file', - type=str, - default=None, - required=False, - help='File of the data x random auto-correlation') - - parser.add_argument('--RD-file', - type=str, - default=None, - required=False, - help='File of the random x data auto-correlation') - - parser.add_argument('--xDD-file', - type=str, - default=None, - required=False, - help='File of the data_1 x data_2 cross-correlation') + description=( + "Export auto and cross-correlation of catalog of objects " "for the fitter." + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--DD-file", + type=str, + default=None, + required=False, + help="File of the data x data auto-correlation", + ) + + parser.add_argument( + "--RR-file", + type=str, + default=None, + required=False, + help="File of the random x random auto-correlation", + ) + + parser.add_argument( + "--DR-file", + type=str, + default=None, + required=False, + help="File of the data x random auto-correlation", + ) parser.add_argument( - '--xRR-file', + "--RD-file", type=str, default=None, required=False, - help='File of the random_1 x random_2 cross-correlation') + help="File of the random x data auto-correlation", + ) - parser.add_argument('--xD1R2-file', - type=str, - default=None, - required=False, - help='File of the data_1 x random_2 cross-correlation') + parser.add_argument( + "--xDD-file", + type=str, + default=None, + required=False, + help="File of the data_1 x data_2 cross-correlation", + ) + + parser.add_argument( + "--xRR-file", + type=str, + default=None, + required=False, + help="File of the random_1 x random_2 cross-correlation", + ) + + parser.add_argument( + "--xD1R2-file", + type=str, + default=None, + required=False, + help="File of the data_1 x random_2 cross-correlation", + ) - parser.add_argument('--xR1D2-file', - type=str, - default=None, - required=False, - help='File of the random_1 x data_2 cross-correlation') + parser.add_argument( + "--xR1D2-file", + type=str, + default=None, + required=False, + help="File of the random_1 x data_2 cross-correlation", + ) parser.add_argument( - '--do-not-smooth-cov', - action='store_true', + "--do-not-smooth-cov", + action="store_true", default=False, - help='Do not smooth the covariance matrix from sub-sampling') + help="Do not smooth the covariance matrix from sub-sampling", + ) - parser.add_argument('--get-cov-from-poisson', - action='store_true', - default=False, - help='Get covariance matrix from Poisson statistics') + parser.add_argument( + "--get-cov-from-poisson", + action="store_true", + default=False, + help="Get covariance matrix from Poisson statistics", + ) parser.add_argument( - '--cov', + "--cov", type=str, default=None, required=False, - help=('Path to a covariance matrix file (if not provided it will be ' - 'calculated by subsampling or from Poisson statistics)')) + help=( + "Path to a covariance matrix file (if not provided it will be " + "calculated by subsampling or from Poisson statistics)" + ), + ) args = parser.parse_args(cmdargs) ### Auto or cross correlation? - if ((args.DD_file is None and args.xDD_file is None) or - (args.DD_file is not None and args.xDD_file is not None) or - (args.cov is not None and not args.get_cov_from_poisson)): - userprint(('ERROR: No data files, or both auto and cross data files, ' - 'or two different method for covariance')) + if ( + (args.DD_file is None and args.xDD_file is None) + or (args.DD_file is not None and args.xDD_file is not None) + or (args.cov is not None and not args.get_cov_from_poisson) + ): + userprint( + ( + "ERROR: No data files, or both auto and cross data files, " + "or two different method for covariance" + ) + ) sys.exit() elif args.DD_file is not None: - corr = 'AUTO' + corr = "AUTO" correlation_files = { - 'DD': args.DD_file, - 'RR': args.RR_file, - 'DR': args.DR_file, - 'RD': args.RD_file + "DD": args.DD_file, + "RR": args.RR_file, + "DR": args.DR_file, + "RD": args.RD_file, } elif not args.xDD_file is None: # TODO: Test if picca_co.py and export_co.py work for cross - corr = 'CROSS' + corr = "CROSS" correlation_files = { - 'xDD': args.xDD_file, - 'xRR': args.xRR_file, - 'xD1R2': args.xD1R2_file, - 'xR1D2': args.xR1D2_file + "xDD": args.xDD_file, + "xRR": args.xRR_file, + "xD1R2": args.xD1R2_file, + "xR1D2": args.xR1D2_file, } # Read files @@ -124,245 +152,271 @@ def main(cmdargs): for type_corr, filename in correlation_files.items(): hdul = fitsio.FITS(filename) header = hdul[1].read_header() - fid_Om = header['OMEGAM'] - fid_Or = header['OMEGAR'] - fid_Ok = header['OMEGAK'] - fid_wl = header['WL'] - if type_corr in ['DD', 'RR']: - num_objects = header['NOBJ'] + fid_Om = header["OMEGAM"] + fid_Or = header["OMEGAR"] + fid_Ok = header["OMEGAK"] + fid_wl = header["WL"] + if type_corr in ["DD", "RR"]: + num_objects = header["NOBJ"] coef = num_objects * (num_objects - 1) else: - num_objects = header['NOBJ'] - num_objects2 = header['NOBJ2'] + num_objects = header["NOBJ"] + num_objects2 = header["NOBJ2"] coef = num_objects * num_objects2 - if type_corr in ['DD', 'xDD']: - data['COEF'] = coef - for item in ['NT', 'NP', 'RTMAX', 'RPMIN', 'RPMAX']: + if type_corr in ["DD", "xDD"]: + data["COEF"] = coef + for item in ["NT", "NP", "RTMAX", "RPMIN", "RPMAX"]: data[item] = header[item] - for item in ['RP', 'RT', 'Z', 'NB']: + for item in ["RP", "RT", "Z", "NB"]: data[item] = np.array(hdul[1][item][:]) data[type_corr] = {} - data[type_corr]['NSIDE'] = header['NSIDE'] - data[type_corr]['HLPXSCHM'] = hdul[2].read_header()['HLPXSCHM'] - w = np.array(hdul[2]['WE'][:]).sum(axis=1) > 0. + data[type_corr]["NSIDE"] = header["NSIDE"] + data[type_corr]["HLPXSCHM"] = hdul[2].read_header()["HLPXSCHM"] + w = np.array(hdul[2]["WE"][:]).sum(axis=1) > 0.0 if w.sum() != w.size: - userprint("INFO: {} sub-samples were empty".format(w.size - - w.sum())) - data[type_corr]['HEALPID'] = hdul[2]['HEALPID'][:][w] - data[type_corr]['WE'] = hdul[2]['WE'][:][w] / coef + userprint("INFO: {} sub-samples were empty".format(w.size - w.sum())) + data[type_corr]["HEALPID"] = hdul[2]["HEALPID"][:][w] + data[type_corr]["WE"] = hdul[2]["WE"][:][w] / coef hdul.close() # Compute correlation - if corr == 'AUTO': - xi_data_data = data['DD']['WE'].sum(axis=0) - xi_random_random = data['RR']['WE'].sum(axis=0) - xi_data_random = data['DR']['WE'].sum(axis=0) - xi_random_data = data['RD']['WE'].sum(axis=0) - w = xi_random_random > 0. + if corr == "AUTO": + xi_data_data = data["DD"]["WE"].sum(axis=0) + xi_random_random = data["RR"]["WE"].sum(axis=0) + xi_data_random = data["DR"]["WE"].sum(axis=0) + xi_random_data = data["RD"]["WE"].sum(axis=0) + w = xi_random_random > 0.0 xi = np.zeros(xi_data_data.size) - xi[w] = (xi_data_data[w] + xi_random_random[w] - xi_random_data[w] - - xi_data_random[w]) / xi_random_random[w] + xi[w] = ( + xi_data_data[w] + + xi_random_random[w] + - xi_random_data[w] + - xi_data_random[w] + ) / xi_random_random[w] else: - xi_data_data = data['xDD']['WE'].sum(axis=0) - xi_random_random = data['xRR']['WE'].sum(axis=0) - xi_data1_random2 = data['xD1R2']['WE'].sum(axis=0) - xi_data2_random1 = data['xR1D2']['WE'].sum(axis=0) - w = xi_random_random > 0. + xi_data_data = data["xDD"]["WE"].sum(axis=0) + xi_random_random = data["xRR"]["WE"].sum(axis=0) + xi_data1_random2 = data["xD1R2"]["WE"].sum(axis=0) + xi_data2_random1 = data["xR1D2"]["WE"].sum(axis=0) + w = xi_random_random > 0.0 xi = np.zeros(xi_data_data.size) - xi[w] = (xi_data_data[w] + xi_random_random[w] - xi_data1_random2[w] - - xi_data2_random1[w]) / xi_random_random[w] - data['DA'] = xi - data['corr_DD'] = xi_data_data - data['corr_RR'] = xi_random_random + xi[w] = ( + xi_data_data[w] + + xi_random_random[w] + - xi_data1_random2[w] + - xi_data2_random1[w] + ) / xi_random_random[w] + data["DA"] = xi + data["corr_DD"] = xi_data_data + data["corr_RR"] = xi_random_random # Compute covariance matrix if not args.cov is None: - userprint('INFO: Read covariance from file') + userprint("INFO: Read covariance from file") hdul = fitsio.FITS(args.cov) - data['CO'] = hdul[1]['CO'][:] + data["CO"] = hdul[1]["CO"][:] hdul.close() elif args.get_cov_from_poisson: - userprint('INFO: Compute covariance from Poisson statistics') - w = data['corr_RR'] > 0. - covariance = np.zeros(data['corr_DD'].size) - covariance[w] = ((data['COEF'] / 2. * data['corr_DD'][w])**2 / - (data['COEF'] / 2. * data['corr_RR'][w])**3) - data['CO'] = np.diag(covariance) + userprint("INFO: Compute covariance from Poisson statistics") + w = data["corr_RR"] > 0.0 + covariance = np.zeros(data["corr_DD"].size) + covariance[w] = (data["COEF"] / 2.0 * data["corr_DD"][w]) ** 2 / ( + data["COEF"] / 2.0 * data["corr_RR"][w] + ) ** 3 + data["CO"] = np.diag(covariance) else: - userprint('INFO: Compute covariance from sub-sampling') + userprint("INFO: Compute covariance from sub-sampling") ### To have same number of HEALPix for type_corr1 in list(correlation_files): for type_corr2 in list(correlation_files): - if data[type_corr1]['NSIDE'] != data[type_corr2]['NSIDE']: - userprint("ERROR: NSIDE are different: {} != " - "{}".format(data[type_corr1]['NSIDE'], - data[type_corr2]['NSIDE'])) + if data[type_corr1]["NSIDE"] != data[type_corr2]["NSIDE"]: + userprint( + "ERROR: NSIDE are different: {} != " + "{}".format( + data[type_corr1]["NSIDE"], data[type_corr2]["NSIDE"] + ) + ) sys.exit() - if data[type_corr1]['HLPXSCHM'] != data[type_corr2]['HLPXSCHM']: - userprint("ERROR: HLPXSCHM are different: {} != " - "{}".format(data[type_corr1]['HLPXSCHM'], - data[type_corr2]['HLPXSCHM'])) + if data[type_corr1]["HLPXSCHM"] != data[type_corr2]["HLPXSCHM"]: + userprint( + "ERROR: HLPXSCHM are different: {} != " + "{}".format( + data[type_corr1]["HLPXSCHM"], data[type_corr2]["HLPXSCHM"] + ) + ) sys.exit() w = np.logical_not( - np.isin(data[type_corr1]['HEALPID'], - data[type_corr2]['HEALPID'])) + np.isin(data[type_corr1]["HEALPID"], data[type_corr2]["HEALPID"]) + ) if w.sum() != 0: - userprint("WARNING: HEALPID are different by {} for {}:{} " - "and {}:{}".format( - w.sum(), type_corr1, - data[type_corr1]['HEALPID'].size, type_corr2, - data[type_corr2]['HEALPID'].size)) - new_healpix = data[type_corr1]['HEALPID'][w] + userprint( + "WARNING: HEALPID are different by {} for {}:{} " + "and {}:{}".format( + w.sum(), + type_corr1, + data[type_corr1]["HEALPID"].size, + type_corr2, + data[type_corr2]["HEALPID"].size, + ) + ) + new_healpix = data[type_corr1]["HEALPID"][w] num_new_healpix = new_healpix.size - num_bins = data[type_corr2]['WE'].shape[1] - data[type_corr2]['HEALPID'] = np.append( - data[type_corr2]['HEALPID'], new_healpix) - data[type_corr2]['WE'] = np.append(data[type_corr2]['WE'], - np.zeros( - (num_new_healpix, - num_bins)), - axis=0) + num_bins = data[type_corr2]["WE"].shape[1] + data[type_corr2]["HEALPID"] = np.append( + data[type_corr2]["HEALPID"], new_healpix + ) + data[type_corr2]["WE"] = np.append( + data[type_corr2]["WE"], + np.zeros((num_new_healpix, num_bins)), + axis=0, + ) # Sort the data by the healpix values for type_corr1 in list(correlation_files): - sort = np.array(data[type_corr1]['HEALPID']).argsort() - data[type_corr1]['WE'] = data[type_corr1]['WE'][sort] - data[type_corr1]['HEALPID'] = data[type_corr1]['HEALPID'][sort] - - if corr == 'AUTO': - xi_data_data = data['DD']['WE'] - xi_random_random = data['RR']['WE'] - xi_data_random = data['DR']['WE'] - xi_random_data = data['RD']['WE'] - w = xi_random_random > 0. + sort = np.array(data[type_corr1]["HEALPID"]).argsort() + data[type_corr1]["WE"] = data[type_corr1]["WE"][sort] + data[type_corr1]["HEALPID"] = data[type_corr1]["HEALPID"][sort] + + if corr == "AUTO": + xi_data_data = data["DD"]["WE"] + xi_random_random = data["RR"]["WE"] + xi_data_random = data["DR"]["WE"] + xi_random_data = data["RD"]["WE"] + w = xi_random_random > 0.0 xi = np.zeros(xi_data_data.shape) - xi[w] = (xi_data_data[w] + xi_random_random[w] - xi_data_random[w] - - xi_random_data[w]) / xi_random_random[w] - weights = data['DD']['WE'] + xi[w] = ( + xi_data_data[w] + + xi_random_random[w] + - xi_data_random[w] + - xi_random_data[w] + ) / xi_random_random[w] + weights = data["DD"]["WE"] else: - xi_data_data = data['xDD']['WE'] - xi_random_random = data['xRR']['WE'] - xi_data1_random2 = data['xD1R2']['WE'] - xi_data2_random1 = data['xR1D2']['WE'] - w = xi_random_random > 0. + xi_data_data = data["xDD"]["WE"] + xi_random_random = data["xRR"]["WE"] + xi_data1_random2 = data["xD1R2"]["WE"] + xi_data2_random1 = data["xR1D2"]["WE"] + w = xi_random_random > 0.0 xi = np.zeros(xi_data_data.shape) - xi[w] = ((xi_data_data[w] + xi_random_random[w] - - xi_data1_random2[w] - xi_data2_random1[w]) / - xi_random_random[w]) - weights = data['xDD']['WE'] - data['HLP_DA'] = xi - data['HLP_WE'] = weights + xi[w] = ( + xi_data_data[w] + + xi_random_random[w] + - xi_data1_random2[w] + - xi_data2_random1[w] + ) / xi_random_random[w] + weights = data["xDD"]["WE"] + data["HLP_DA"] = xi + data["HLP_WE"] = weights if args.do_not_smooth_cov: - userprint('INFO: The covariance will not be smoothed') + userprint("INFO: The covariance will not be smoothed") covariance = compute_cov(xi, weights) else: - userprint('INFO: The covariance will be smoothed') - delta_r_par = (data['RPMAX'] - data['RPMIN']) / data['NP'] - delta_r_trans = (data['RTMAX'] - 0.) / data['NT'] - covariance = smooth_cov(xi, - weights, - data['RP'], - data['RT'], - delta_r_par=delta_r_par, - delta_r_trans=delta_r_trans) - data['CO'] = covariance + userprint("INFO: The covariance will be smoothed") + delta_r_par = (data["RPMAX"] - data["RPMIN"]) / data["NP"] + delta_r_trans = (data["RTMAX"] - 0.0) / data["NT"] + covariance = smooth_cov( + xi, + weights, + data["RP"], + data["RT"], + delta_r_par=delta_r_par, + delta_r_trans=delta_r_trans, + ) + data["CO"] = covariance try: - scipy.linalg.cholesky(data['CO']) + scipy.linalg.cholesky(data["CO"]) except scipy.linalg.LinAlgError: - userprint('WARNING: Matrix is not positive definite') + userprint("WARNING: Matrix is not positive definite") # Identity distortion matrix - data['DM'] = np.eye(data['DA'].size) + data["DM"] = np.eye(data["DA"].size) # Save results - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = {} - if corr == 'AUTO': - nside = data['DD']['NSIDE'] + if corr == "AUTO": + nside = data["DD"]["NSIDE"] else: - nside = data['xDD']['NSIDE'] - header = [{ - 'name': 'RPMIN', - 'value': data['RPMIN'], - 'comment': 'Minimum r-parallel' - }, { - 'name': 'RPMAX', - 'value': data['RPMAX'], - 'comment': 'Maximum r-parallel' - }, { - 'name': 'RTMAX', - 'value': data['RTMAX'], - 'comment': 'Maximum r-transverse' - }, { - 'name': 'NP', - 'value': data['NP'], - 'comment': 'Number of bins in r-parallel' - }, { - 'name': 'NT', - 'value': data['NT'], - 'comment': 'Number of bins in r-transverse' - }, { - 'name': 'NSIDE', - 'value': nside, - 'comment': 'Healpix nside' - }, { - 'name': 'OMEGAM', - 'value': fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - } + nside = data["xDD"]["NSIDE"] + header = [ + {"name": "RPMIN", "value": data["RPMIN"], "comment": "Minimum r-parallel"}, + {"name": "RPMAX", "value": data["RPMAX"], "comment": "Maximum r-parallel"}, + {"name": "RTMAX", "value": data["RTMAX"], "comment": "Maximum r-transverse"}, + {"name": "NP", "value": data["NP"], "comment": "Number of bins in r-parallel"}, + { + "name": "NT", + "value": data["NT"], + "comment": "Number of bins in r-transverse", + }, + {"name": "NSIDE", "value": nside, "comment": "Healpix nside"}, + { + "name": "OMEGAM", + "value": fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, ] - names = ['RP', 'RT', 'Z', 'DA', 'CO', 'DM', 'NB'] + names = ["RP", "RT", "Z", "DA", "CO", "DM", "NB"] comment = [ - 'R-parallel', 'R-transverse', 'Redshift', 'Correlation', - 'Covariance matrix', 'Distortion matrix', 'Number of pairs' + "R-parallel", + "R-transverse", + "Redshift", + "Correlation", + "Covariance matrix", + "Distortion matrix", + "Number of pairs", ] - results.write([data[name] for name in names], - names=names, - header=header, - comment=comment, - extname='COR') + results.write( + [data[name] for name in names], + names=names, + header=header, + comment=comment, + extname="COR", + ) if args.cov is None and not args.get_cov_from_poisson: - if corr == 'AUTO': - healpix_scheme = data['DD']['HLPXSCHM'] - healpix_list = data['DD']['HEALPID'] + if corr == "AUTO": + healpix_scheme = data["DD"]["HLPXSCHM"] + healpix_list = data["DD"]["HEALPID"] else: - healpix_scheme = data['xDD']['HLPXSCHM'] - healpix_list = data['xDD']['HEALPID'] - header2 = [{ - 'name': 'HLPXSCHM', - 'value': healpix_scheme, - 'comment': 'healpix scheme' - }] - comment = ['Healpix index', 'Sum of weight', 'Correlation'] - results.write([healpix_list, data['HLP_WE'], data['HLP_DA']], - names=['HEALPID', 'WE', 'DA'], - header=header2, - comment=comment, - extname='SUB_COR') + healpix_scheme = data["xDD"]["HLPXSCHM"] + healpix_list = data["xDD"]["HEALPID"] + header2 = [ + {"name": "HLPXSCHM", "value": healpix_scheme, "comment": "healpix scheme"} + ] + comment = ["Healpix index", "Sum of weight", "Correlation"] + results.write( + [healpix_list, data["HLP_WE"], data["HLP_DA"]], + names=["HEALPID", "WE", "DA"], + header=header2, + comment=comment, + extname="SUB_COR", + ) results.close() -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_export_cross_covariance.py b/py/picca/bin/picca_export_cross_covariance.py old mode 100644 new mode 100755 index f5375a474..9a7ecd9a4 --- a/py/picca/bin/picca_export_cross_covariance.py +++ b/py/picca/bin/picca_export_cross_covariance.py @@ -1,39 +1,41 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the cross-covariance matrix between two correlations.""" import argparse import sys + +import fitsio import numpy as np import scipy.linalg -import fitsio from picca.utils import compute_cov, userprint + def main(cmdargs): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the cross-covariance matrix between two ' - 'correlations')) + description=("Compute the cross-covariance matrix between two " "correlations"), + ) parser.add_argument( - '--data1', + "--data1", type=str, default=None, required=True, - help='Correlation 1 produced via picca_cf.py, picca_xcf.py, ...') + help="Correlation 1 produced via picca_cf.py, picca_xcf.py, ...", + ) parser.add_argument( - '--data2', + "--data2", type=str, default=None, required=True, - help='Correlation 2 produced via picca_cf.py, picca_xcf.py, ...') + help="Correlation 2 produced via picca_cf.py, picca_xcf.py, ...", + ) - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) args = parser.parse_args(cmdargs) @@ -43,71 +45,78 @@ def main(cmdargs): for index, filename in enumerate([args.data1, args.data2]): hdul = fitsio.FITS(filename) header = hdul[1].read_header() - nside = header['NSIDE'] + nside = header["NSIDE"] header2 = hdul[2].read_header() - healpix_scheme = header2['HLPXSCHM'] - weights = np.array(hdul[2]['WE'][:]) - healpix_list = np.array(hdul[2]['HEALPID'][:]) + healpix_scheme = header2["HLPXSCHM"] + weights = np.array(hdul[2]["WE"][:]) + healpix_list = np.array(hdul[2]["HEALPID"][:]) - if 'DA_BLIND' in hdul[2].get_colnames(): - xi = np.array(hdul[2]['DA_BLIND'][:]) + if "DA_BLIND" in hdul[2].get_colnames(): + xi = np.array(hdul[2]["DA_BLIND"][:]) else: - xi = np.array(hdul[2]['DA'][:]) + xi = np.array(hdul[2]["DA"][:]) data[index] = { - 'DA': xi, - 'WE': weights, - 'HEALPID': healpix_list, - 'NSIDE': nside, - 'HLPXSCHM': healpix_scheme + "DA": xi, + "WE": weights, + "HEALPID": healpix_list, + "NSIDE": nside, + "HLPXSCHM": healpix_scheme, } hdul.close() # exit if NSIDE1 != NSIDE2 - if data[0]['NSIDE'] != data[1]['NSIDE']: - userprint(("ERROR: NSIDE are different: {} != " - "{}").format(data[0]['NSIDE'], data[1]['NSIDE'])) + if data[0]["NSIDE"] != data[1]["NSIDE"]: + userprint( + ("ERROR: NSIDE are different: {} != " "{}").format( + data[0]["NSIDE"], data[1]["NSIDE"] + ) + ) sys.exit() # exit if HLPXSCHM1 != HLPXSCHM2 - if data[0]['HLPXSCHM'] != data[1]['HLPXSCHM']: - userprint(("ERROR: HLPXSCHM are different: {} != " - "{}").format(data[0]['HLPXSCHM'], data[1]['HLPXSCHM'])) + if data[0]["HLPXSCHM"] != data[1]["HLPXSCHM"]: + userprint( + ("ERROR: HLPXSCHM are different: {} != " "{}").format( + data[0]["HLPXSCHM"], data[1]["HLPXSCHM"] + ) + ) sys.exit() # Add unshared healpix as empty data for key in sorted(list(data.keys())): key2 = (key + 1) % 2 - w = np.logical_not(np.isin(data[key2]['HEALPID'], data[key]['HEALPID'])) + w = np.logical_not(np.isin(data[key2]["HEALPID"], data[key]["HEALPID"])) if w.sum() > 0: - new_healpix = data[key2]['HEALPID'][w] + new_healpix = data[key2]["HEALPID"][w] num_new_healpix = new_healpix.size - num_bins = data[key]['DA'].shape[1] - userprint(("Some healpix are unshared in data {}: " - "{}").format(key, new_healpix)) - data[key]['DA'] = np.append(data[key]['DA'], - np.zeros((num_new_healpix, num_bins)), - axis=0) - data[key]['WE'] = np.append(data[key]['WE'], - np.zeros((num_new_healpix, num_bins)), - axis=0) - data[key]['HEALPID'] = np.append(data[key]['HEALPID'], new_healpix) + num_bins = data[key]["DA"].shape[1] + userprint( + ("Some healpix are unshared in data {}: " "{}").format(key, new_healpix) + ) + data[key]["DA"] = np.append( + data[key]["DA"], np.zeros((num_new_healpix, num_bins)), axis=0 + ) + data[key]["WE"] = np.append( + data[key]["WE"], np.zeros((num_new_healpix, num_bins)), axis=0 + ) + data[key]["HEALPID"] = np.append(data[key]["HEALPID"], new_healpix) # Sort the data by the healpix values for key in sorted(list(data.keys())): - sort = np.array(data[key]['HEALPID']).argsort() - data[key]['DA'] = data[key]['DA'][sort] - data[key]['WE'] = data[key]['WE'][sort] - data[key]['HEALPID'] = data[key]['HEALPID'][sort] + sort = np.array(data[key]["HEALPID"]).argsort() + data[key]["DA"] = data[key]["DA"][sort] + data[key]["WE"] = data[key]["WE"][sort] + data[key]["HEALPID"] = data[key]["HEALPID"][sort] # Append the data - xi = np.append(data[0]['DA'], data[1]['DA'], axis=1) - weights = np.append(data[0]['WE'], data[1]['WE'], axis=1) + xi = np.append(data[0]["DA"], data[1]["DA"], axis=1) + weights = np.append(data[0]["WE"], data[1]["WE"], axis=1) # Compute the covariance covariance = compute_cov(xi, weights) # Get the cross-covariance - num_bins = data[0]['DA'].shape[1] + num_bins = data[0]["DA"].shape[1] cross_covariance = covariance.copy() cross_covariance = cross_covariance[:, num_bins:] cross_covariance = cross_covariance[:num_bins, :] @@ -123,17 +132,19 @@ def main(cmdargs): try: scipy.linalg.cholesky(covariance) except scipy.linalg.LinAlgError: - userprint('WARNING: Matrix is not positive definite') + userprint("WARNING: Matrix is not positive definite") ### Save - results = fitsio.FITS(args.out, 'rw', clobber=True) - results.write([cross_covariance, cross_correlation], - names=['CO', 'COR'], - comment=['Covariance matrix', 'Correlation matrix'], - extname='COVAR') + results = fitsio.FITS(args.out, "rw", clobber=True) + results.write( + [cross_covariance, cross_correlation], + names=["CO", "COR"], + comment=["Covariance matrix", "Correlation matrix"], + extname="COVAR", + ) results.close() -if __name__ == '__main__': - cmdargs=sys.argv[1:] - main(cmdargs) \ No newline at end of file +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_fast_metal_dmat.py b/py/picca/bin/picca_fast_metal_dmat.py old mode 100644 new mode 100755 index 01a2fbd2f..1b2764b70 --- a/py/picca/bin/picca_fast_metal_dmat.py +++ b/py/picca/bin/picca_fast_metal_dmat.py @@ -1,17 +1,18 @@ -#!/usr/bin/env python -"""Compute the metal matrices -""" -import sys +#!/usr/bin/env python3 +"""Compute the metal matrices""" +import argparse import os +import sys import time -import argparse -import numpy as np + import fitsio +import numpy as np -from picca import constants, cf, io +from picca import cf, constants, io from picca.utils import userprint -DEFAULT_SI_METALS = ['SiIII(1207)','SiII(1190)','SiII(1193)','SiII(1260)'] +DEFAULT_SI_METALS = ["SiIII(1207)", "SiII(1190)", "SiII(1193)", "SiII(1260)"] + def read_stack_deltas_table(filename): """ @@ -25,13 +26,15 @@ def read_stack_deltas_table(filename): return fitsio.read(filename, "STACK_DELTAS") -def calc_fast_metal_dmat(in_lambda_abs_1, - in_lambda_abs_2, - out_lambda_abs_1, - out_lambda_abs_2, - stack_table_1, - stack_table_2, - rebin_factor=None): +def calc_fast_metal_dmat( + in_lambda_abs_1, + in_lambda_abs_2, + out_lambda_abs_1, + out_lambda_abs_2, + stack_table_1, + stack_table_2, + rebin_factor=None, +): """Computes the metal distortion matrix. Args: @@ -74,41 +77,53 @@ def calc_fast_metal_dmat(in_lambda_abs_1, weight2 = stack_table_2["WEIGHT"] if rebin_factor is not None: size1 = loglam1.size - loglam1 = loglam1[:(size1 // rebin_factor) * rebin_factor].reshape( - (size1 // rebin_factor), rebin_factor).mean(-1) - weight1 = weight1[:(size1 // rebin_factor) * rebin_factor].reshape( - (size1 // rebin_factor), rebin_factor).mean(-1) + loglam1 = ( + loglam1[: (size1 // rebin_factor) * rebin_factor] + .reshape((size1 // rebin_factor), rebin_factor) + .mean(-1) + ) + weight1 = ( + weight1[: (size1 // rebin_factor) * rebin_factor] + .reshape((size1 // rebin_factor), rebin_factor) + .mean(-1) + ) size2 = loglam2.size - loglam2 = loglam2[:(size2 // rebin_factor) * rebin_factor].reshape( - (size2 // rebin_factor), rebin_factor).mean(-1) - weight2 = weight2[:(size2 // rebin_factor) * rebin_factor].reshape( - (size2 // rebin_factor), rebin_factor).mean(-1) + loglam2 = ( + loglam2[: (size2 // rebin_factor) * rebin_factor] + .reshape((size2 // rebin_factor), rebin_factor) + .mean(-1) + ) + weight2 = ( + weight2[: (size2 // rebin_factor) * rebin_factor] + .reshape((size2 // rebin_factor), rebin_factor) + .mean(-1) + ) # input - input_z1 = (10**loglam1) / constants.ABSORBER_IGM[in_lambda_abs_1] - 1. - input_z2 = (10**loglam2) / constants.ABSORBER_IGM[in_lambda_abs_2] - 1. + input_z1 = (10**loglam1) / constants.ABSORBER_IGM[in_lambda_abs_1] - 1.0 + input_z2 = (10**loglam2) / constants.ABSORBER_IGM[in_lambda_abs_2] - 1.0 input_r1 = cf.cosmo.get_r_comov(input_z1) input_r2 = cf.cosmo.get_r_comov(input_z2) # all pairs input_rp = ( - input_r1[:, None] - - input_r2[None, :]).ravel() # same sign as line 676 of cf.py (1-2) + input_r1[:, None] - input_r2[None, :] + ).ravel() # same sign as line 676 of cf.py (1-2) if not cf.x_correlation: input_rp = np.abs(input_rp) - input_dist = ((input_r1[:, None]+input_r2[None, :])/2).ravel() + input_dist = ((input_r1[:, None] + input_r2[None, :]) / 2).ravel() # output - output_z1 = (10**loglam1) / constants.ABSORBER_IGM[out_lambda_abs_1] - 1. - output_z2 = (10**loglam2) / constants.ABSORBER_IGM[out_lambda_abs_2] - 1. + output_z1 = (10**loglam1) / constants.ABSORBER_IGM[out_lambda_abs_1] - 1.0 + output_z2 = (10**loglam2) / constants.ABSORBER_IGM[out_lambda_abs_2] - 1.0 output_r1 = cf.cosmo.get_r_comov(output_z1) output_r2 = cf.cosmo.get_r_comov(output_z2) # all pairs output_rp = ( - output_r1[:, None] - - output_r2[None, :]).ravel() # same sign as line 676 of cf.py (1-2) + output_r1[:, None] - output_r2[None, :] + ).ravel() # same sign as line 676 of cf.py (1-2) if not cf.x_correlation: output_rp = np.abs(output_rp) - output_dist = ((output_r1[:, None]+output_r2[None, :])/2).ravel() + output_dist = ((output_r1[:, None] + output_r2[None, :]) / 2).ravel() # weights # alpha_in: in (1+z)^(alpha_in-1) is a scaling used to model how the metal contribution @@ -123,25 +138,25 @@ def calc_fast_metal_dmat(in_lambda_abs_1, # so here we have to apply both scalings (in the original code : # alpha_in is applied in cf.calc_metal_dmat and alpha_out in io.read_deltas) weights = ( - (weight1 * ((1 + input_z1)**(alpha_in_1 + alpha_out_1 - 2)))[:, None] * - (weight2 * - ((1 + input_z2)**(alpha_in_2 + alpha_out_2 - 2)))[None, :]).ravel() + (weight1 * ((1 + input_z1) ** (alpha_in_1 + alpha_out_1 - 2)))[:, None] + * (weight2 * ((1 + input_z2) ** (alpha_in_2 + alpha_out_2 - 2)))[None, :] + ).ravel() # r_par distortion matrix - rpbins = cf.r_par_min \ - + (cf.r_par_max-cf.r_par_min)/cf.num_bins_r_par*np.arange(cf.num_bins_r_par+1) + rpbins = cf.r_par_min + ( + cf.r_par_max - cf.r_par_min + ) / cf.num_bins_r_par * np.arange(cf.num_bins_r_par + 1) # I checked the orientation of the matrix - rp_1d_dmat, _, _ = np.histogram2d(output_rp, - input_rp, - bins=(rpbins, rpbins), - weights=weights) + rp_1d_dmat, _, _ = np.histogram2d( + output_rp, input_rp, bins=(rpbins, rpbins), weights=weights + ) # normalize - sum_rp_1d_dmat = np.sum(rp_1d_dmat,axis=0) - rp_1d_dmat /= (sum_rp_1d_dmat+(sum_rp_1d_dmat==0)) + sum_rp_1d_dmat = np.sum(rp_1d_dmat, axis=0) + rp_1d_dmat /= sum_rp_1d_dmat + (sum_rp_1d_dmat == 0) # independently, we compute the r_trans distortion matrix - rtbins = cf.r_trans_max/cf.num_bins_r_trans*np.arange(cf.num_bins_r_trans+1) + rtbins = cf.r_trans_max / cf.num_bins_r_trans * np.arange(cf.num_bins_r_trans + 1) # we have input_dist , output_dist and weight. # we don't need to store the absolute comoving distances but the ratio between output and input. # we rebin that to compute the rest faster. @@ -151,26 +166,40 @@ def calc_fast_metal_dmat(in_lambda_abs_1, # solid angle contibuting for each distance propto theta_max**2 = (r_trans_max/dist)**2 propto 1/dist**2 # we weight the distances with this additional factor # using the input or the output distance in the solid angle weight gives virtually the same result - #distance_ratio_weights,distance_ratio_bins = np.histogram(output_dist/input_dist,bins=4*rtbins.size,weights=weights/input_dist**2*(input_rpcf.r_par_min)) + # distance_ratio_weights,distance_ratio_bins = np.histogram(output_dist/input_dist,bins=4*rtbins.size,weights=weights/input_dist**2*(input_rpcf.r_par_min)) # we also select only distance ratio for which the input_rp (that of the true separation of the absorbers) is small, so that this # fast matrix calculation is accurate where it matters the most - distance_ratio_weights,distance_ratio_bins = np.histogram(output_dist/input_dist,bins=4*rtbins.size,weights=weights/input_dist**2*(np.abs(input_rp)<20.)) - distance_ratios=(distance_ratio_bins[1:]+distance_ratio_bins[:-1])/2. + distance_ratio_weights, distance_ratio_bins = np.histogram( + output_dist / input_dist, + bins=4 * rtbins.size, + weights=weights / input_dist**2 * (np.abs(input_rp) < 20.0), + ) + distance_ratios = (distance_ratio_bins[1:] + distance_ratio_bins[:-1]) / 2.0 # now we need to scan as a function of separation angles, or equivalently rt. - rt_bin_centers = (rtbins[:-1]+rtbins[1:])/2. - rt_bin_half_size = (rtbins[1]-rtbins[0])/2. + rt_bin_centers = (rtbins[:-1] + rtbins[1:]) / 2.0 + rt_bin_half_size = (rtbins[1] - rtbins[0]) / 2.0 # we are oversampling the correlation function rt grid to correctly compute bin migration. - oversample = 7 - delta_rt = np.linspace(-rt_bin_half_size,rt_bin_half_size*(1-2./oversample),oversample)[None,:] # the -2/oversample term is needed to get a even-spaced grid - rt_1d_dmat = np.zeros((cf.num_bins_r_trans,cf.num_bins_r_trans)) - for i,rt in enumerate(rt_bin_centers) : + oversample = 7 + delta_rt = np.linspace( + -rt_bin_half_size, rt_bin_half_size * (1 - 2.0 / oversample), oversample + )[ + None, : + ] # the -2/oversample term is needed to get a even-spaced grid + rt_1d_dmat = np.zeros((cf.num_bins_r_trans, cf.num_bins_r_trans)) + for i, rt in enumerate(rt_bin_centers): # the weight is proportional to rt+delta_rt to get the correct solid angle effect inside the bin (but it's almost a negligible effect) - rt_1d_dmat[:,i],_ = np.histogram((distance_ratios[:,None]*(rt+delta_rt)[None,:]).ravel(),bins=rtbins,weights=(distance_ratio_weights[:,None]*(rt+delta_rt)[None,:]).ravel()) + rt_1d_dmat[:, i], _ = np.histogram( + (distance_ratios[:, None] * (rt + delta_rt)[None, :]).ravel(), + bins=rtbins, + weights=( + distance_ratio_weights[:, None] * (rt + delta_rt)[None, :] + ).ravel(), + ) # normalize - sum_rt_1d_dmat = np.sum(rt_1d_dmat,axis=0) - rt_1d_dmat /= (sum_rt_1d_dmat+(sum_rt_1d_dmat==0)) + sum_rt_1d_dmat = np.sum(rt_1d_dmat, axis=0) + rt_1d_dmat /= sum_rt_1d_dmat + (sum_rt_1d_dmat == 0) # now that we have both distortion along r_par and r_trans, we have to combine them # we just multiply the two matrices, with indices splitted for rt and rp @@ -178,40 +207,44 @@ def calc_fast_metal_dmat(in_lambda_abs_1, # rt_index = full_index%cf.num_bins_r_trans # rp_index = full_index//cf.num_bins_r_trans num_bins_total = cf.num_bins_r_par * cf.num_bins_r_trans - dmat = np.zeros((num_bins_total,num_bins_total)) - pp = np.arange(num_bins_total,dtype=int) - for k in range(num_bins_total) : - dmat[k,pp] = rt_1d_dmat[k%cf.num_bins_r_trans,pp%cf.num_bins_r_trans] * rp_1d_dmat[k//cf.num_bins_r_trans,pp//cf.num_bins_r_trans] + dmat = np.zeros((num_bins_total, num_bins_total)) + pp = np.arange(num_bins_total, dtype=int) + for k in range(num_bins_total): + dmat[k, pp] = ( + rt_1d_dmat[k % cf.num_bins_r_trans, pp % cf.num_bins_r_trans] + * rp_1d_dmat[k // cf.num_bins_r_trans, pp // cf.num_bins_r_trans] + ) # compute effective z,rp,rt - sum_out_weight, _ = np.histogram(output_rp, bins=rpbins, weights=weights) - sum_out_weight_rp, _ = np.histogram(output_rp, - bins=rpbins, - weights=weights * - (output_rp[None, :].ravel())) + sum_out_weight, _ = np.histogram(output_rp, bins=rpbins, weights=weights) + sum_out_weight_rp, _ = np.histogram( + output_rp, bins=rpbins, weights=weights * (output_rp[None, :].ravel()) + ) # return the redshift of the actual absorber, which is the average of input_z1 # and input_z2 sum_out_weight_z, _ = np.histogram( output_rp, bins=rpbins, - weights=weights * - (((input_z1[:, None] + input_z2[None, :]) / 2.).ravel())) + weights=weights * (((input_z1[:, None] + input_z2[None, :]) / 2.0).ravel()), + ) r_par_eff_1d = sum_out_weight_rp / (sum_out_weight + (sum_out_weight == 0)) - z_eff_1d = sum_out_weight_z / (sum_out_weight + (sum_out_weight == 0)) + z_eff_1d = sum_out_weight_z / (sum_out_weight + (sum_out_weight == 0)) # r_trans has no weights here - r1 = np.arange(cf.num_bins_r_trans) * cf.r_trans_max / cf.num_bins_r_trans - r2 = (1+np.arange(cf.num_bins_r_trans)) * cf.r_trans_max / cf.num_bins_r_trans - r_trans_eff_1d = (2*(r2**3-r1**3))/(3*(r2**2-r1**2)) # this is to account for the solid angle effect on the mean + r1 = np.arange(cf.num_bins_r_trans) * cf.r_trans_max / cf.num_bins_r_trans + r2 = (1 + np.arange(cf.num_bins_r_trans)) * cf.r_trans_max / cf.num_bins_r_trans + r_trans_eff_1d = (2 * (r2**3 - r1**3)) / ( + 3 * (r2**2 - r1**2) + ) # this is to account for the solid angle effect on the mean full_index = np.arange(num_bins_total) - rt_index = full_index%cf.num_bins_r_trans - rp_index = full_index//cf.num_bins_r_trans + rt_index = full_index % cf.num_bins_r_trans + rp_index = full_index // cf.num_bins_r_trans - r_par_eff_2d = r_par_eff_1d[rp_index] - z_eff_2d = z_eff_1d[rp_index] + r_par_eff_2d = r_par_eff_1d[rp_index] + z_eff_2d = z_eff_1d[rp_index] r_trans_eff_2d = r_trans_eff_1d[rt_index] return dmat, r_par_eff_2d, r_trans_eff_2d, z_eff_2d @@ -223,215 +256,255 @@ def main(cmdargs=None): absorption.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description='Computes metal matrices') + description="Computes metal matrices", + ) - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) parser.add_argument( - '-i', - '--in-attributes', + "-i", + "--in-attributes", type=str, default=None, required=True, - help='Path to delta_attributes.fits.gz file with hdu STACK_DELTAS' - ' containing table with at least rows "LOGLAM" and "WEIGHT"') + help="Path to delta_attributes.fits.gz file with hdu STACK_DELTAS" + ' containing table with at least rows "LOGLAM" and "WEIGHT"', + ) - parser.add_argument('--in-attributes2', - type=str, - default=None, - required=False, - help='Path to 2nd delta_attributes.fits.gz file') + parser.add_argument( + "--in-attributes2", + type=str, + default=None, + required=False, + help="Path to 2nd delta_attributes.fits.gz file", + ) parser.add_argument( - '--delta-dir', + "--delta-dir", type=str, default=None, required=False, - help='Path to directory with delta*.gz to get the blinding info' - ' (default is trying to guess from attributes file)') - - parser.add_argument('--rp-min', - type=float, - default=0., - required=False, - help='Min r-parallel [h^-1 Mpc]') - - parser.add_argument('--rp-max', - type=float, - default=200., - required=False, - help='Max r-parallel [h^-1 Mpc]') - - parser.add_argument('--rt-max', - type=float, - default=200., - required=False, - help='Max r-transverse [h^-1 Mpc]') - - parser.add_argument('--np', - type=int, - default=50, - required=False, - help='Number of r-parallel bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of r-transverse bins') + help="Path to directory with delta*.gz to get the blinding info" + " (default is trying to guess from attributes file)", + ) + + parser.add_argument( + "--rp-min", + type=float, + default=0.0, + required=False, + help="Min r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rp-max", + type=float, + default=200.0, + required=False, + help="Max r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rt-max", + type=float, + default=200.0, + required=False, + help="Max r-transverse [h^-1 Mpc]", + ) parser.add_argument( - '--coef-binning-model', + "--np", type=int, default=50, required=False, help="Number of r-parallel bins" + ) + + parser.add_argument( + "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" + ) + + parser.add_argument( + "--coef-binning-model", type=int, default=1, required=False, - help=('Coefficient multiplying np and nt to get finner binning for the ' - 'model')) + help=( + "Coefficient multiplying np and nt to get finner binning for the " "model" + ), + ) + + parser.add_argument( + "--z-cut-min", + type=float, + default=0.0, + required=False, + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--z-cut-min', + "--z-cut-max", type=float, - default=0., + default=10.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) + + parser.add_argument( + "--z-min-sources", + type=float, + default=0.0, + required=False, + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--z-cut-max', + "--z-max-sources", type=float, - default=10., + default=10.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) - - parser.add_argument('--z-min-sources', - type=float, - default=0., - required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) - - parser.add_argument('--z-max-sources', - type=float, - default=10., - required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--lambda-abs', + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) parser.add_argument( - '--lambda-abs2', + "--lambda-abs2", type=str, default=None, required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the 2nd delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the 2nd delta" + ), + ) parser.add_argument( - '--abs-igm', + "--abs-igm", type=str, default=[], required=True, - nargs='*', - help=('List of names of metal absorption in picca.constants present in ' - 'forest')) + nargs="*", + help=( + "List of names of metal absorption in picca.constants present in " "forest" + ), + ) parser.add_argument( - '--abs-igm2', + "--abs-igm2", type=str, default=[], required=False, - nargs='*', - help=('List of names of metal absorption in picca.constants present in ' - '2nd forest')) + nargs="*", + help=( + "List of names of metal absorption in picca.constants present in " + "2nd forest" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol', + "--z-evol", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol2', + "--z-evol2", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the 2nd delta field') + help="Exponent of the redshift evolution of the 2nd delta field", + ) parser.add_argument( - '--metal-alpha', + "--metal-alpha", type=float, - default=1., + default=1.0, required=False, - help='Exponent of the redshift evolution of the metal delta field') + help="Exponent of the redshift evolution of the metal delta field", + ) parser.add_argument( - '--fid-Om', + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--unfold-cf', - action='store_true', + "--unfold-cf", + action="store_true", required=False, - help='rp can be positive or negative depending on the relative ' - 'position between absorber1 and absorber2') + help="rp can be positive or negative depending on the relative " + "position between absorber1 and absorber2", + ) parser.add_argument( - '--rebin-factor', + "--rebin-factor", type=int, default=None, required=False, - help='Rebin factor for deltas. If not None, deltas will ' - 'be rebinned by that factor') + help="Rebin factor for deltas. If not None, deltas will " + "be rebinned by that factor", + ) - parser.add_argument('--relevant-metals', - action='store_true', - required=False, - help='compute only the metal correlations used by Vega' - 'i.e. 4 LyaxSi matrices and CIVxCIV') + parser.add_argument( + "--relevant-metals", + action="store_true", + required=False, + help="compute only the metal correlations used by Vega" + "i.e. 4 LyaxSi matrices and CIVxCIV", + ) args = parser.parse_args(cmdargs) @@ -469,10 +542,12 @@ def main(cmdargs=None): blinding = io.read_blinding(args.delta_dir) # load fiducial cosmology - cf.cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl,) + cf.cosmo = constants.Cosmo( + Om=args.fid_Om, + Or=args.fid_Or, + Ok=args.fid_Ok, + wl=args.fid_wl, + ) t0 = time.time() @@ -485,9 +560,7 @@ def main(cmdargs=None): stack_table2 = stack_table1 # reference to first one t1 = time.time() - userprint( - f'picca_fast_metal_dmat.py - Time reading data: {(t1-t0)/60:.3f} minutes' - ) + userprint(f"picca_fast_metal_dmat.py - Time reading data: {(t1-t0)/60:.3f} minutes") abs_igm = [args.lambda_abs] + args.abs_igm userprint(f"abs_igm = {abs_igm}") @@ -518,8 +591,9 @@ def main(cmdargs=None): continue if args.relevant_metals: - if not (abs_igm1 == "LYA" and abs_igm2 in DEFAULT_SI_METALS) \ - and not (abs_igm1 == "CIV(eff)" and abs_igm1 == abs_igm2): + if not (abs_igm1 == "LYA" and abs_igm2 in DEFAULT_SI_METALS) and not ( + abs_igm1 == "CIV(eff)" and abs_igm1 == abs_igm2 + ): continue userprint("Computing", abs_igm1, abs_igm2) @@ -532,7 +606,8 @@ def main(cmdargs=None): args.lambda_abs2, stack_table1, stack_table2, - rebin_factor=args.rebin_factor) + rebin_factor=args.rebin_factor, + ) # add these results to the list ofor the different metal absorption dmat_all.append(dmat) @@ -543,82 +618,93 @@ def main(cmdargs=None): t2 = time.time() userprint( - f'picca_fast_metal_dmat.py - Time computing all metal matrices : {(t2-t1)/60:.3f} minutes' + f"picca_fast_metal_dmat.py - Time computing all metal matrices : {(t2-t1)/60:.3f} minutes" ) # save the results - results = fitsio.FITS(args.out, 'rw', clobber=True) - header = [{ - 'name': 'RPMIN', - 'value': cf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RPMAX', - 'value': cf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RTMAX', - 'value': cf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' - }, { - 'name': 'NP', - 'value': cf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' - }, { - 'name': 'NT', - 'value': cf.num_bins_r_trans, - 'comment': ' Number of bins in r-transverse' - }, { - 'name': 'COEFMOD', - 'value': args.coef_binning_model, - 'comment': 'Coefficient for model binning' - }, { - 'name': 'ZCUTMIN', - 'value': cf.z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, { - 'name': 'ZCUTMAX', - 'value': cf.z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, { - 'name': 'REJ', - 'value': cf.reject, - 'comment': 'Rejection factor' - }, { - 'name': 'ALPHAMET', - 'value': args.metal_alpha, - 'comment': 'Evolution of metal bias' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': - 'WL', - 'value': - args.fid_wl, - 'comment': - 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - }] + results = fitsio.FITS(args.out, "rw", clobber=True) + header = [ + { + "name": "RPMIN", + "value": cf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": cf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", + }, + { + "name": "RTMAX", + "value": cf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", + }, + { + "name": "NP", + "value": cf.num_bins_r_par, + "comment": "Number of bins in r-parallel", + }, + { + "name": "NT", + "value": cf.num_bins_r_trans, + "comment": " Number of bins in r-transverse", + }, + { + "name": "COEFMOD", + "value": args.coef_binning_model, + "comment": "Coefficient for model binning", + }, + { + "name": "ZCUTMIN", + "value": cf.z_cut_min, + "comment": "Minimum redshift of pairs", + }, + { + "name": "ZCUTMAX", + "value": cf.z_cut_max, + "comment": "Maximum redshift of pairs", + }, + {"name": "REJ", "value": cf.reject, "comment": "Rejection factor"}, + { + "name": "ALPHAMET", + "value": args.metal_alpha, + "comment": "Evolution of metal bias", + }, + { + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + ] len_names = np.array([len(name) for name in names]).max() - names = np.array(names, dtype='S' + str(len_names)) - results.write([np.array(names)], - names=['ABS_IGM'], - header=header, - comment=['Absorption name'], - extname='ATTRI') + names = np.array(names, dtype="S" + str(len_names)) + results.write( + [np.array(names)], + names=["ABS_IGM"], + header=header, + comment=["Absorption name"], + extname="ATTRI", + ) dmat_name = "DM_" if blinding != "none": @@ -629,38 +715,40 @@ def main(cmdargs=None): out_comment = [] out_units = [] for index, name in enumerate(names): - out_names += ['RP_' + name] + out_names += ["RP_" + name] out_list += [r_par_all[index]] - out_comment += ['R-parallel'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-parallel"] + out_units += ["h^-1 Mpc"] - out_names += ['RT_' + name] + out_names += ["RT_" + name] out_list += [r_trans_all[index]] - out_comment += ['R-transverse'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-transverse"] + out_units += ["h^-1 Mpc"] - out_names += ['Z_' + name] + out_names += ["Z_" + name] out_list += [z_all[index]] - out_comment += ['Redshift'] - out_units += [''] + out_comment += ["Redshift"] + out_units += [""] out_names += [dmat_name + name] out_list += [dmat_all[index]] - out_comment += ['Distortion matrix'] - out_units += [''] - - #out_names += ['WDM_' + name] - #out_list += [weights_dmat_all[index]] - #out_comment += ['Sum of weight'] - #out_units += [''] - - results.write(out_list, - names=out_names, - comment=out_comment, - units=out_units, - extname='MDMAT') + out_comment += ["Distortion matrix"] + out_units += [""] + + # out_names += ['WDM_' + name] + # out_list += [weights_dmat_all[index]] + # out_comment += ['Sum of weight'] + # out_units += [''] + + results.write( + out_list, names=out_names, comment=out_comment, units=out_units, extname="MDMAT" + ) results.close() t3 = time.time() - userprint( - f'picca_fast_metal_dmat.py - Time total : {(t3-t0)/60:.3f} minutes') + userprint(f"picca_fast_metal_dmat.py - Time total : {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_fast_metal_xdmat.py b/py/picca/bin/picca_fast_metal_xdmat.py old mode 100644 new mode 100755 index d8c812be0..1191e4601 --- a/py/picca/bin/picca_fast_metal_xdmat.py +++ b/py/picca/bin/picca_fast_metal_xdmat.py @@ -1,14 +1,14 @@ -#!/usr/bin/env python -"""Compute the metal matrices -""" -import sys +#!/usr/bin/env python3 +"""Compute the metal matrices""" +import argparse import os +import sys import time -import argparse -import numpy as np + import fitsio +import numpy as np -from picca import constants, xcf, io +from picca import constants, io, xcf from picca.utils import userprint @@ -24,12 +24,9 @@ def read_stack_deltas_table(filename): return fitsio.read(filename, "STACK_DELTAS") -def calc_fast_metal_dmat(in_lambda_abs, - out_lambda_abs, - stack_table, - z_qso, - weight_qso, - rebin_factor=None): +def calc_fast_metal_dmat( + in_lambda_abs, out_lambda_abs, stack_table, z_qso, weight_qso, rebin_factor=None +): """Computes the metal distortion matrix. Args: @@ -58,35 +55,40 @@ def calc_fast_metal_dmat(in_lambda_abs, weight_forest = stack_table["WEIGHT"] if rebin_factor is not None: size = loglam.size - loglam = loglam[:(size // rebin_factor) * rebin_factor].reshape( - (size // rebin_factor), rebin_factor).mean(-1) - weight_forest = weight_forest[:(size // rebin_factor) * - rebin_factor].reshape( - (size // rebin_factor), - rebin_factor).mean(-1) + loglam = ( + loglam[: (size // rebin_factor) * rebin_factor] + .reshape((size // rebin_factor), rebin_factor) + .mean(-1) + ) + weight_forest = ( + weight_forest[: (size // rebin_factor) * rebin_factor] + .reshape((size // rebin_factor), rebin_factor) + .mean(-1) + ) # input input_zf = (10**loglam) / constants.ABSORBER_IGM[ - in_lambda_abs] - 1. # redshift in the forest + in_lambda_abs + ] - 1.0 # redshift in the forest input_rf = xcf.cosmo.get_r_comov(input_zf) r_qso = xcf.cosmo.get_r_comov(z_qso) # comoving distance for qso # all pairs input_rp = ( - input_rf[:, None] - - r_qso[None, :]).ravel() # same sign as line 528 of xcf.py (forest-qso) - input_dist = ((input_rf[:, None]+r_qso[None, :])/2).ravel() + input_rf[:, None] - r_qso[None, :] + ).ravel() # same sign as line 528 of xcf.py (forest-qso) + input_dist = ((input_rf[:, None] + r_qso[None, :]) / 2).ravel() # output - output_zf = (10**loglam) / constants.ABSORBER_IGM[out_lambda_abs] - 1. + output_zf = (10**loglam) / constants.ABSORBER_IGM[out_lambda_abs] - 1.0 output_rf = xcf.cosmo.get_r_comov(output_zf) # all pairs output_rp = ( - output_rf[:, None] - - r_qso[None, :]).ravel() # same sign as line 528 of xcf.py (forest-qso) - output_dist = ((output_rf[:, None]+r_qso[None, :])/2).ravel() + output_rf[:, None] - r_qso[None, :] + ).ravel() # same sign as line 528 of xcf.py (forest-qso) + output_dist = ((output_rf[:, None] + r_qso[None, :]) / 2).ravel() # weights # alpha_in: in (1+z)^(alpha_in-1) is a scaling used to model how the metal contribution @@ -98,26 +100,28 @@ def calc_fast_metal_dmat(in_lambda_abs, # so here we have to apply both scalings (in the original code : alpha_in is applied in # xcf.calc_metal_dmat and alpha_out in io.read_deltas) # qso weights have already been scaled with (1+z)^alpha_obj - weights = ((weight_forest * - ((1 + input_zf)**(alpha_in + alpha_out - 2)))[:, None] * - weight_qso[None, :]).ravel() + weights = ( + (weight_forest * ((1 + input_zf) ** (alpha_in + alpha_out - 2)))[:, None] + * weight_qso[None, :] + ).ravel() # r_par distortion matrix rpbins = xcf.r_par_min + ( - xcf.r_par_max - - xcf.r_par_min) / xcf.num_bins_r_par * np.arange(xcf.num_bins_r_par + 1) + xcf.r_par_max - xcf.r_par_min + ) / xcf.num_bins_r_par * np.arange(xcf.num_bins_r_par + 1) # I checked the orientation of the matrix - rp_1d_dmat, _, _ = np.histogram2d(output_rp, - input_rp, - bins=(rpbins, rpbins), - weights=weights) + rp_1d_dmat, _, _ = np.histogram2d( + output_rp, input_rp, bins=(rpbins, rpbins), weights=weights + ) # normalize - sum_rp_1d_dmat = np.sum(rp_1d_dmat,axis=0) - rp_1d_dmat /= (sum_rp_1d_dmat+(sum_rp_1d_dmat==0)) + sum_rp_1d_dmat = np.sum(rp_1d_dmat, axis=0) + rp_1d_dmat /= sum_rp_1d_dmat + (sum_rp_1d_dmat == 0) # independently, we compute the r_trans distortion matrix - rtbins = xcf.r_trans_max/xcf.num_bins_r_trans*np.arange(xcf.num_bins_r_trans+1) + rtbins = ( + xcf.r_trans_max / xcf.num_bins_r_trans * np.arange(xcf.num_bins_r_trans + 1) + ) # we have input_dist , output_dist and weight. # we don't need to store the absolute comoving distances but the ratio between output and input. # we rebin that to compute the rest faster. @@ -129,23 +133,37 @@ def calc_fast_metal_dmat(in_lambda_abs, # using the input or the output distance in the solid angle weight gives virtually the same result # we also select only distance ratio for which the input_rp (that of the true separation of the absorbers) is small, so that this # fast matrix calculation is accurate where it matters the most - distance_ratio_weights,distance_ratio_bins = np.histogram(output_dist/input_dist,bins=4*rtbins.size,weights=weights/input_dist**2*(np.abs(input_rp)<20.)) - distance_ratios=(distance_ratio_bins[1:]+distance_ratio_bins[:-1])/2. + distance_ratio_weights, distance_ratio_bins = np.histogram( + output_dist / input_dist, + bins=4 * rtbins.size, + weights=weights / input_dist**2 * (np.abs(input_rp) < 20.0), + ) + distance_ratios = (distance_ratio_bins[1:] + distance_ratio_bins[:-1]) / 2.0 # now we need to scan as a function of separation angles, or equivalently rt. - rt_bin_centers = (rtbins[:-1]+rtbins[1:])/2. - rt_bin_half_size = (rtbins[1]-rtbins[0])/2. + rt_bin_centers = (rtbins[:-1] + rtbins[1:]) / 2.0 + rt_bin_half_size = (rtbins[1] - rtbins[0]) / 2.0 # we are oversampling the correlation function rt grid to correctly compute bin migration. - oversample = 7 - delta_rt = np.linspace(-rt_bin_half_size,rt_bin_half_size*(1-2./oversample),oversample)[None,:] # the -2/oversample term is needed to get a even-spaced grid - rt_1d_dmat = np.zeros((xcf.num_bins_r_trans,xcf.num_bins_r_trans)) - for i,rt in enumerate(rt_bin_centers) : + oversample = 7 + delta_rt = np.linspace( + -rt_bin_half_size, rt_bin_half_size * (1 - 2.0 / oversample), oversample + )[ + None, : + ] # the -2/oversample term is needed to get a even-spaced grid + rt_1d_dmat = np.zeros((xcf.num_bins_r_trans, xcf.num_bins_r_trans)) + for i, rt in enumerate(rt_bin_centers): # the weight is proportional to rt to get the correct solid angle effect (it's almost a negligible effect) - rt_1d_dmat[:,i],_ = np.histogram((distance_ratios[:,None]*(rt+delta_rt)[None,:]).ravel(),bins=rtbins,weights=(distance_ratio_weights[:,None]*(rt+delta_rt)[None,:]).ravel()) + rt_1d_dmat[:, i], _ = np.histogram( + (distance_ratios[:, None] * (rt + delta_rt)[None, :]).ravel(), + bins=rtbins, + weights=( + distance_ratio_weights[:, None] * (rt + delta_rt)[None, :] + ).ravel(), + ) # normalize - sum_rt_1d_dmat = np.sum(rt_1d_dmat,axis=0) - rt_1d_dmat /= (sum_rt_1d_dmat+(sum_rt_1d_dmat==0)) + sum_rt_1d_dmat = np.sum(rt_1d_dmat, axis=0) + rt_1d_dmat /= sum_rt_1d_dmat + (sum_rt_1d_dmat == 0) # now that we have both distortion along r_par and r_trans, we have to combine them # we just multiply the two matrices, with indices splitted for rt and rp @@ -153,41 +171,44 @@ def calc_fast_metal_dmat(in_lambda_abs, # rt_index = full_index%xcf.num_bins_r_trans # rp_index = full_index//xcf.num_bins_r_trans num_bins_total = xcf.num_bins_r_par * xcf.num_bins_r_trans - dmat = np.zeros((num_bins_total,num_bins_total)) - pp = np.arange(num_bins_total,dtype=int) - for k in range(num_bins_total) : - dmat[k,pp] = rt_1d_dmat[k%xcf.num_bins_r_trans,pp%xcf.num_bins_r_trans] * rp_1d_dmat[k//xcf.num_bins_r_trans,pp//xcf.num_bins_r_trans] - + dmat = np.zeros((num_bins_total, num_bins_total)) + pp = np.arange(num_bins_total, dtype=int) + for k in range(num_bins_total): + dmat[k, pp] = ( + rt_1d_dmat[k % xcf.num_bins_r_trans, pp % xcf.num_bins_r_trans] + * rp_1d_dmat[k // xcf.num_bins_r_trans, pp // xcf.num_bins_r_trans] + ) # mean outputs sum_out_weight, _ = np.histogram(output_rp, bins=rpbins, weights=weights) - sum_out_weight_rp, _ = np.histogram(output_rp, - bins=rpbins, - weights=weights * - (output_rp[None, :].ravel())) + sum_out_weight_rp, _ = np.histogram( + output_rp, bins=rpbins, weights=weights * (output_rp[None, :].ravel()) + ) # return the redshift of the actual absorber, which is the average of input_zf # and z_qso sum_out_weight_z, _ = np.histogram( output_rp, bins=rpbins, - weights=weights * - (((input_zf[:, None] + z_qso[None, :]) / 2.).ravel())) + weights=weights * (((input_zf[:, None] + z_qso[None, :]) / 2.0).ravel()), + ) r_par_eff_1d = sum_out_weight_rp / (sum_out_weight + (sum_out_weight == 0)) - z_eff_1d = sum_out_weight_z / (sum_out_weight + (sum_out_weight == 0)) + z_eff_1d = sum_out_weight_z / (sum_out_weight + (sum_out_weight == 0)) # r_trans has no weights here - r1 = np.arange(xcf.num_bins_r_trans) * xcf.r_trans_max / xcf.num_bins_r_trans - r2 = (1+np.arange(xcf.num_bins_r_trans)) * xcf.r_trans_max / xcf.num_bins_r_trans - r_trans_eff_1d = (2*(r2**3-r1**3))/(3*(r2**2-r1**2)) # this is to account for the solid angle effect on the mean + r1 = np.arange(xcf.num_bins_r_trans) * xcf.r_trans_max / xcf.num_bins_r_trans + r2 = (1 + np.arange(xcf.num_bins_r_trans)) * xcf.r_trans_max / xcf.num_bins_r_trans + r_trans_eff_1d = (2 * (r2**3 - r1**3)) / ( + 3 * (r2**2 - r1**2) + ) # this is to account for the solid angle effect on the mean full_index = np.arange(num_bins_total) - rt_index = full_index%xcf.num_bins_r_trans - rp_index = full_index//xcf.num_bins_r_trans + rt_index = full_index % xcf.num_bins_r_trans + rp_index = full_index // xcf.num_bins_r_trans - r_par_eff_2d = r_par_eff_1d[rp_index] - z_eff_2d = z_eff_1d[rp_index] + r_par_eff_2d = r_par_eff_1d[rp_index] + z_eff_2d = z_eff_1d[rp_index] r_trans_eff_2d = r_trans_eff_1d[rt_index] return dmat, r_par_eff_2d, r_trans_eff_2d, z_eff_2d @@ -195,218 +216,260 @@ def calc_fast_metal_dmat(in_lambda_abs, def main(cmdargs=None): """Compute the metal matrix of the cross-correlation delta x object for - a list of IGM absorption.""" + a list of IGM absorption.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Computes metal matrices for the cross-correlation ' - 'delta x object for a list of IGM absorption.')) + description=( + "Computes metal matrices for the cross-correlation " + "delta x object for a list of IGM absorption." + ), + ) - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) parser.add_argument( - '-i', - '--in-attributes', + "-i", + "--in-attributes", type=str, default=None, required=True, - help='Path to delta_attributes.fits.gz file with hdu STACK_DELTAS' - ' containing table with at least rows "LOGLAM" and "WEIGHT"') + help="Path to delta_attributes.fits.gz file with hdu STACK_DELTAS" + ' containing table with at least rows "LOGLAM" and "WEIGHT"', + ) parser.add_argument( - '--delta-dir', + "--delta-dir", type=str, default=None, required=False, - help='Path to directory with delta*.gz to get the blinding info' - ' (default is trying to guess from attributes file)') - - parser.add_argument('--drq', - type=str, - default=None, - required=True, - help='Catalog of objects in DRQ format') - - parser.add_argument('--mode', - type=str, - default='sdss', - choices=['sdss', 'desi', 'desi_mocks', 'desi_healpix'], - required=False, - help='type of catalog supplied, default sdss') - - parser.add_argument('--rp-min', - type=float, - default=-200., - required=False, - help='Min r-parallel [h^-1 Mpc]') - - parser.add_argument('--rp-max', - type=float, - default=200., - required=False, - help='Max r-parallel [h^-1 Mpc]') - - parser.add_argument('--rt-max', - type=float, - default=200., - required=False, - help='Max r-transverse [h^-1 Mpc]') - - parser.add_argument('--np', - type=int, - default=100, - required=False, - help='Number of r-parallel bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of r-transverse bins') + help="Path to directory with delta*.gz to get the blinding info" + " (default is trying to guess from attributes file)", + ) + + parser.add_argument( + "--drq", + type=str, + default=None, + required=True, + help="Catalog of objects in DRQ format", + ) + + parser.add_argument( + "--mode", + type=str, + default="sdss", + choices=["sdss", "desi", "desi_mocks", "desi_healpix"], + required=False, + help="type of catalog supplied, default sdss", + ) + + parser.add_argument( + "--rp-min", + type=float, + default=-200.0, + required=False, + help="Min r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rp-max", + type=float, + default=200.0, + required=False, + help="Max r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rt-max", + type=float, + default=200.0, + required=False, + help="Max r-transverse [h^-1 Mpc]", + ) parser.add_argument( - '--coef-binning-model', + "--np", type=int, default=100, required=False, help="Number of r-parallel bins" + ) + + parser.add_argument( + "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" + ) + + parser.add_argument( + "--coef-binning-model", type=int, default=1, required=False, - help=('Coefficient multiplying np and nt to get finner binning for the ' - 'model')) + help=( + "Coefficient multiplying np and nt to get finner binning for the " "model" + ), + ) - parser.add_argument('--z-min-obj', - type=float, - default=0, - required=False, - help='Min redshift for object field') + parser.add_argument( + "--z-min-obj", + type=float, + default=0, + required=False, + help="Min redshift for object field", + ) + + parser.add_argument( + "--z-max-obj", + type=float, + default=10, + required=False, + help="Max redshift for object field", + ) + + parser.add_argument( + "--z-cut-min", + type=float, + default=0.0, + required=False, + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) - parser.add_argument('--z-max-obj', - type=float, - default=10, - required=False, - help='Max redshift for object field') + parser.add_argument( + "--z-cut-max", + type=float, + default=10.0, + required=False, + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) parser.add_argument( - '--z-cut-min', + "--z-min-sources", type=float, - default=0., + default=0.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--z-cut-max', + "--z-max-sources", type=float, - default=10., + default=10.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) - - parser.add_argument('--z-min-sources', - type=float, - default=0., - required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) - - parser.add_argument('--z-max-sources', - type=float, - default=10., - required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--lambda-abs', + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) - parser.add_argument('--obj-name', - type=str, - default='QSO', - required=False, - help='Name of the object tracer') + parser.add_argument( + "--obj-name", + type=str, + default="QSO", + required=False, + help="Name of the object tracer", + ) parser.add_argument( - '--abs-igm', + "--abs-igm", type=str, default=None, required=False, - nargs='*', - help='List of names of metal absorption in picca.constants') + nargs="*", + help="List of names of metal absorption in picca.constants", + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol-del', + "--z-evol-del", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol-obj', + "--z-evol-obj", type=float, default=1.44, required=False, - help='Exponent of the redshift evolution of the object field') + help="Exponent of the redshift evolution of the object field", + ) parser.add_argument( - '--metal-alpha', + "--metal-alpha", type=float, - default=1., + default=1.0, required=False, - help='Exponent of the redshift evolution of the metal delta field') + help="Exponent of the redshift evolution of the metal delta field", + ) parser.add_argument( - '--fid-Om', + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--rebin-factor', + "--rebin-factor", type=int, default=None, required=False, - help='Rebin factor for deltas. If not None, deltas will ' - 'be rebinned by that factor') - parser.add_argument('--qso-z-bins', - type=int, - default=1000, - required=False, - help='Bins for the distribution of QSO redshifts') + help="Rebin factor for deltas. If not None, deltas will " + "be rebinned by that factor", + ) + parser.add_argument( + "--qso-z-bins", + type=int, + default=1000, + required=False, + help="Bins for the distribution of QSO redshifts", + ) args = parser.parse_args(cmdargs) @@ -441,10 +504,12 @@ def main(cmdargs=None): blinding = io.read_blinding(args.delta_dir) # load fiducial cosmology - cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl,) + cosmo = constants.Cosmo( + Om=args.fid_Om, + Or=args.fid_Or, + Ok=args.fid_Ok, + wl=args.fid_wl, + ) xcf.cosmo = cosmo t0 = time.time() @@ -453,13 +518,15 @@ def main(cmdargs=None): stack_table = read_stack_deltas_table(args.in_attributes) # read objets - catalog = io.read_drq(args.drq, - z_min=args.z_min_obj, - z_max=args.z_max_obj, - keep_bal=True, - mode=args.mode) - z_qso = catalog['Z'] - weight_qso = ((1. + z_qso) / (1. + args.z_ref))**(args.z_evol_obj - 1.) + catalog = io.read_drq( + args.drq, + z_min=args.z_min_obj, + z_max=args.z_max_obj, + keep_bal=True, + mode=args.mode, + ) + z_qso = catalog["Z"] + weight_qso = ((1.0 + z_qso) / (1.0 + args.z_ref)) ** (args.z_evol_obj - 1.0) zbins = args.qso_z_bins userprint(f"Use histogram of QSO redshifts with {zbins} bins") @@ -470,8 +537,7 @@ def main(cmdargs=None): weight_qso = histo_w[selection] t1 = time.time() - userprint( - f'picca_metal_xdmat.py - Time reading data: {(t1-t0)/60:.3f} minutes') + userprint(f"picca_metal_xdmat.py - Time reading data: {(t1-t0)/60:.3f} minutes") # intitialize arrays to store the results for the different metal absorption dmat_all = [] @@ -492,7 +558,8 @@ def main(cmdargs=None): stack_table, z_qso, weight_qso, - rebin_factor=args.rebin_factor) + rebin_factor=args.rebin_factor, + ) # add these results to the list ofor the different metal absorption dmat_all.append(dmat) @@ -503,75 +570,87 @@ def main(cmdargs=None): t2 = time.time() userprint( - f'picca_metal_xdmat.py - Time computing all metal matrix: {(t2-t1)/60:.3f} minutes' + f"picca_metal_xdmat.py - Time computing all metal matrix: {(t2-t1)/60:.3f} minutes" ) # save the results - results = fitsio.FITS(args.out, 'rw', clobber=True) - header = [{ - 'name': 'RPMIN', - 'value': xcf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RPMAX', - 'value': xcf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RTMAX', - 'value': xcf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' - }, { - 'name': 'NP', - 'value': xcf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' - }, { - 'name': 'NT', - 'value': xcf.num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' - }, { - 'name': 'COEFMOD', - 'value': args.coef_binning_model, - 'comment': 'Coefficient for model binning' - }, { - 'name': 'ZCUTMIN', - 'value': xcf.z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, { - 'name': 'ZCUTMAX', - 'value': xcf.z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': - 'WL', - 'value': - args.fid_wl, - 'comment': - 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - }] + results = fitsio.FITS(args.out, "rw", clobber=True) + header = [ + { + "name": "RPMIN", + "value": xcf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": xcf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", + }, + { + "name": "RTMAX", + "value": xcf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", + }, + { + "name": "NP", + "value": xcf.num_bins_r_par, + "comment": "Number of bins in r-parallel", + }, + { + "name": "NT", + "value": xcf.num_bins_r_trans, + "comment": "Number of bins in r-transverse", + }, + { + "name": "COEFMOD", + "value": args.coef_binning_model, + "comment": "Coefficient for model binning", + }, + { + "name": "ZCUTMIN", + "value": xcf.z_cut_min, + "comment": "Minimum redshift of pairs", + }, + { + "name": "ZCUTMAX", + "value": xcf.z_cut_max, + "comment": "Maximum redshift of pairs", + }, + { + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + ] len_names = np.array([len(name) for name in names]).max() - names = np.array(names, dtype='S' + str(len_names)) + names = np.array(names, dtype="S" + str(len_names)) results.write( [np.array(names)], - names=['ABS_IGM'], + names=["ABS_IGM"], header=header, - comment=['Number of pairs', 'Number of used pairs', 'Absorption name'], - extname='ATTRI') + comment=["Number of pairs", "Number of used pairs", "Absorption name"], + extname="ATTRI", + ) dmat_name = "DM_" if blinding != "none": @@ -582,32 +661,35 @@ def main(cmdargs=None): out_comment = [] out_units = [] for index, name in enumerate(names): - out_names += ['RP_' + args.obj_name + '_' + name] + out_names += ["RP_" + args.obj_name + "_" + name] out_list += [r_par_all[index]] - out_comment += ['R-parallel'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-parallel"] + out_units += ["h^-1 Mpc"] - out_names += ['RT_' + args.obj_name + '_' + name] + out_names += ["RT_" + args.obj_name + "_" + name] out_list += [r_trans_all[index]] - out_comment += ['R-transverse'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-transverse"] + out_units += ["h^-1 Mpc"] - out_names += ['Z_' + args.obj_name + '_' + name] + out_names += ["Z_" + args.obj_name + "_" + name] out_list += [z_all[index]] - out_comment += ['Redshift'] - out_units += [''] + out_comment += ["Redshift"] + out_units += [""] - out_names += [dmat_name + args.obj_name + '_' + name] + out_names += [dmat_name + args.obj_name + "_" + name] out_list += [dmat_all[index]] - out_comment += ['Distortion matrix'] - out_units += [''] - - results.write(out_list, - names=out_names, - comment=out_comment, - units=out_units, - extname='MDMAT') + out_comment += ["Distortion matrix"] + out_units += [""] + + results.write( + out_list, names=out_names, comment=out_comment, units=out_units, extname="MDMAT" + ) results.close() t3 = time.time() - userprint(f'picca_metal_xdmat.py - Time total: {(t3-t0)/60:.3f} minutes') + userprint(f"picca_metal_xdmat.py - Time total: {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_metal_dmat.py b/py/picca/bin/picca_metal_dmat.py old mode 100644 new mode 100755 index 874c08436..ba59bd85c --- a/py/picca/bin/picca_metal_dmat.py +++ b/py/picca/bin/picca_metal_dmat.py @@ -1,23 +1,26 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the auto and cross-correlation of delta fields for a list of IGM absorption. This module follow the procedure described in sections 4.3 of du Mas des Bourboux et al. 2020 (In prep) to compute the distortion matrix """ -import time import argparse import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value +import sys +import time from functools import partial -import numpy as np +from multiprocessing import Lock, Value, cpu_count + import fitsio +import numpy as np -from picca import constants, cf, utils, io +from picca import cf, constants, io, utils from picca.utils import userprint -#store the default silicon metals modelled in the CF/XCF -DEFAULT_SI_METALS = ['SiIII(1207)','SiII(1190)','SiII(1193)','SiII(1260)'] +# store the default silicon metals modelled in the CF/XCF +DEFAULT_SI_METALS = ["SiIII(1207)", "SiII(1190)", "SiII(1193)", "SiII(1260)"] + def calc_metal_dmat(abs_igm1, abs_igm2, healpixs): """Computes the metal distortion matrix. @@ -41,9 +44,7 @@ def calc_metal_dmat(abs_igm1, abs_igm2, healpixs): """ cf.fill_neighs(healpixs) np.random.seed(healpixs[0]) - dmat_data = cf.compute_metal_dmat(healpixs, - abs_igm1=abs_igm1, - abs_igm2=abs_igm2) + dmat_data = cf.compute_metal_dmat(healpixs, abs_igm1=abs_igm1, abs_igm2=abs_igm2) return dmat_data @@ -53,238 +54,283 @@ def main(cmdargs=None): absorption.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the auto and cross-correlation of delta fields ' - 'for a list of IGM absorption.')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--in-dir2', - type=str, - default=None, - required=False, - help='Directory to 2nd delta files') - - parser.add_argument('--rp-min', - type=float, - default=0., - required=False, - help='Min r-parallel [h^-1 Mpc]') - - parser.add_argument('--rp-max', - type=float, - default=200., - required=False, - help='Max r-parallel [h^-1 Mpc]') - - parser.add_argument('--rt-max', - type=float, - default=200., - required=False, - help='Max r-transverse [h^-1 Mpc]') - - parser.add_argument('--np', - type=int, - default=50, - required=False, - help='Number of r-parallel bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of r-transverse bins') - - parser.add_argument( - '--coef-binning-model', + description=( + "Compute the auto and cross-correlation of delta fields " + "for a list of IGM absorption." + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--in-dir2", + type=str, + default=None, + required=False, + help="Directory to 2nd delta files", + ) + + parser.add_argument( + "--rp-min", + type=float, + default=0.0, + required=False, + help="Min r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rp-max", + type=float, + default=200.0, + required=False, + help="Max r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rt-max", + type=float, + default=200.0, + required=False, + help="Max r-transverse [h^-1 Mpc]", + ) + + parser.add_argument( + "--np", type=int, default=50, required=False, help="Number of r-parallel bins" + ) + + parser.add_argument( + "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" + ) + + parser.add_argument( + "--coef-binning-model", type=int, default=1, required=False, - help=('Coefficient multiplying np and nt to get finner binning for the ' - 'model')) + help=( + "Coefficient multiplying np and nt to get finner binning for the " "model" + ), + ) parser.add_argument( - '--z-cut-min', + "--z-cut-min", type=float, - default=0., + default=0.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--z-cut-max', + "--z-cut-max", type=float, - default=10., + default=10.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) parser.add_argument( - '--z-min-sources', + "--z-min-sources", type=float, - default=0., + default=0.0, required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--z-max-sources', + "--z-max-sources", type=float, - default=10., + default=10.0, required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--lambda-abs', + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) parser.add_argument( - '--lambda-abs2', + "--lambda-abs2", type=str, default=None, required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the 2nd delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the 2nd delta" + ), + ) parser.add_argument( - '--abs-igm', + "--abs-igm", type=str, default=[], required=False, - nargs='*', - help=('List of names of metal absorption in picca.constants present in ' - 'forest')) + nargs="*", + help=( + "List of names of metal absorption in picca.constants present in " "forest" + ), + ) parser.add_argument( - '--abs-igm2', + "--abs-igm2", type=str, default=[], required=False, - nargs='*', - help=('List of names of metal absorption in picca.constants present in ' - '2nd forest')) + nargs="*", + help=( + "List of names of metal absorption in picca.constants present in " + "2nd forest" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol', + "--z-evol", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol2', + "--z-evol2", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the 2nd delta field') + help="Exponent of the redshift evolution of the 2nd delta field", + ) parser.add_argument( - '--metal-alpha', + "--metal-alpha", type=float, - default=1., + default=1.0, required=False, - help='Exponent of the redshift evolution of the metal delta field') + help="Exponent of the redshift evolution of the metal delta field", + ) parser.add_argument( - '--fid-Om', + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--remove-same-half-plate-close-pairs', - action='store_true', + "--remove-same-half-plate-close-pairs", + action="store_true", required=False, - help='Reject pairs in the first bin in r-parallel from same half plate') + help="Reject pairs in the first bin in r-parallel from same half plate", + ) parser.add_argument( - '--rej', + "--rej", type=float, - default=1., + default=1.0, + required=False, + help=( + "Fraction of rejected forest-forest pairs: -1=no rejection, " + "1=all rejection" + ), + ) + + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) + + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, required=False, - help=('Fraction of rejected forest-forest pairs: -1=no rejection, ' - '1=all rejection')) - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') - - parser.add_argument( - '--unfold-cf', - action='store_true', + help="Maximum number of spectra to read", + ) + + parser.add_argument( + "--unfold-cf", + action="store_true", required=False, - help=('rp can be positive or negative depending on the relative ' - 'position between absorber1 and absorber2')) - - parser.add_argument('--rebin-factor', - type=int, - default=None, - required=False, - help='Rebin factor for deltas. If not None, deltas will ' - 'be rebinned by that factor') - - parser.add_argument('--fast-metals', - action='store_true', - required=False, - help='compute only the metal correlations used by Vega' - 'i.e. 4 LyaxSi matrices and CIVxCIV') + help=( + "rp can be positive or negative depending on the relative " + "position between absorber1 and absorber2" + ), + ) + + parser.add_argument( + "--rebin-factor", + type=int, + default=None, + required=False, + help="Rebin factor for deltas. If not None, deltas will " + "be rebinned by that factor", + ) + + parser.add_argument( + "--fast-metals", + action="store_true", + required=False, + help="compute only the metal correlations used by Vega" + "i.e. 4 LyaxSi matrices and CIVxCIV", + ) args = parser.parse_args(cmdargs) @@ -319,25 +365,29 @@ def main(cmdargs=None): blinding = io.read_blinding(args.in_dir) # load fiducial cosmology - cf.cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl,) + cf.cosmo = constants.Cosmo( + Om=args.fid_Om, + Or=args.fid_Or, + Ok=args.fid_Ok, + wl=args.fid_wl, + ) t0 = time.time() ### Read data 1 - data, num_data, z_min, z_max = io.read_deltas(args.in_dir, - cf.nside, - cf.lambda_abs, - cf.alpha, - cf.z_ref, - cf.cosmo, - max_num_spec=args.nspec, - nproc=args.nproc, - rebin_factor=args.rebin_factor, - z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + data, num_data, z_min, z_max = io.read_deltas( + args.in_dir, + cf.nside, + cf.lambda_abs, + cf.alpha, + cf.z_ref, + cf.cosmo, + max_num_spec=args.nspec, + nproc=args.nproc, + rebin_factor=args.rebin_factor, + z_min_qso=args.z_min_sources, + z_max_qso=args.z_max_sources, + ) del z_max cf.data = data cf.num_data = num_data @@ -371,19 +421,19 @@ def main(cmdargs=None): nproc=args.nproc, rebin_factor=args.rebin_factor, z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + z_max_qso=args.z_max_sources, + ) del z_max2 cf.data2 = data2 cf.num_data2 = num_data2 - cf.ang_max = utils.compute_ang_max(cf.cosmo, cf.r_trans_max, z_min, - z_min2) + cf.ang_max = utils.compute_ang_max(cf.cosmo, cf.r_trans_max, z_min, z_min2) userprint("") userprint("done, npix = {}".format(len(data2))) t1 = time.time() - userprint(f'picca_metal_dmat.py - Time reading data: {(t1-t0)/60:.3f} minutes') + userprint(f"picca_metal_dmat.py - Time reading data: {(t1-t0)/60:.3f} minutes") - cf.counter = Value('i', 0) + cf.counter = Value("i", 0) cf.lock = Lock() cpu_data = {} for index, healpix in enumerate(sorted(list(data.keys()))): @@ -424,25 +474,23 @@ def main(cmdargs=None): continue if args.fast_metals: - if not (abs_igm1 == "LYA" and abs_igm2 in DEFAULT_SI_METALS) \ - and not (abs_igm1 == "CIV(eff)" and abs_igm1 == abs_igm2): + if not (abs_igm1 == "LYA" and abs_igm2 in DEFAULT_SI_METALS) and not ( + abs_igm1 == "CIV(eff)" and abs_igm1 == abs_igm2 + ): continue cf.counter.value = 0 - calc_metal_dmat_wrapper = partial(calc_metal_dmat, abs_igm1, - abs_igm2) + calc_metal_dmat_wrapper = partial(calc_metal_dmat, abs_igm1, abs_igm2) userprint("") # compute the distortion matrix if args.nproc > 1: - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=args.nproc) - dmat_data = pool.map(calc_metal_dmat_wrapper, - sorted(cpu_data.values())) + dmat_data = pool.map(calc_metal_dmat_wrapper, sorted(cpu_data.values())) pool.close() elif args.nproc == 1: - dmat_data = map(calc_metal_dmat_wrapper, - sorted(cpu_data.values())) + dmat_data = map(calc_metal_dmat_wrapper, sorted(cpu_data.values())) # merge the results from different CPUs dmat_data = list(dmat_data) @@ -474,94 +522,94 @@ def main(cmdargs=None): num_pairs_used_all.append(num_pairs_used) t2 = time.time() - userprint(f'picca_metal_dmat.py - Time computing all metal matrices : {(t2-t1)/60:.3f} minutes') + userprint( + f"picca_metal_dmat.py - Time computing all metal matrices : {(t2-t1)/60:.3f} minutes" + ) # save the results - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = [ { - 'name': 'RPMIN', - 'value': cf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' + "name": "RPMIN", + "value": cf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", }, { - 'name': 'RPMAX', - 'value': cf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' + "name": "RPMAX", + "value": cf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", }, { - 'name': 'RTMAX', - 'value': cf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' + "name": "RTMAX", + "value": cf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", }, { - 'name': 'NP', - 'value': cf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' + "name": "NP", + "value": cf.num_bins_r_par, + "comment": "Number of bins in r-parallel", }, { - 'name': 'NT', - 'value': cf.num_bins_r_trans, - 'comment': ' Number of bins in r-transverse' + "name": "NT", + "value": cf.num_bins_r_trans, + "comment": " Number of bins in r-transverse", }, { - 'name': 'COEFMOD', - 'value': args.coef_binning_model, - 'comment': 'Coefficient for model binning' + "name": "COEFMOD", + "value": args.coef_binning_model, + "comment": "Coefficient for model binning", }, { - 'name': 'ZCUTMIN', - 'value': cf.z_cut_min, - 'comment': 'Minimum redshift of pairs' + "name": "ZCUTMIN", + "value": cf.z_cut_min, + "comment": "Minimum redshift of pairs", }, { - 'name': 'ZCUTMAX', - 'value': cf.z_cut_max, - 'comment': 'Maximum redshift of pairs' + "name": "ZCUTMAX", + "value": cf.z_cut_max, + "comment": "Maximum redshift of pairs", }, + {"name": "REJ", "value": cf.reject, "comment": "Rejection factor"}, { - 'name': 'REJ', - 'value': cf.reject, - 'comment': 'Rejection factor' + "name": "ALPHAMET", + "value": args.metal_alpha, + "comment": "Evolution of metal bias", }, { - 'name': 'ALPHAMET', - 'value': args.metal_alpha, - 'comment': 'Evolution of metal bias' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - } - ] + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + ] len_names = np.array([len(name) for name in names]).max() - names = np.array(names, dtype='S' + str(len_names)) + names = np.array(names, dtype="S" + str(len_names)) results.write( - [ - np.array(num_pairs_all), - np.array(num_pairs_used_all), - np.array(names) - ], - names=['NPALL', 'NPUSED', 'ABS_IGM'], + [np.array(num_pairs_all), np.array(num_pairs_used_all), np.array(names)], + names=["NPALL", "NPUSED", "ABS_IGM"], header=header, - comment=['Number of pairs', 'Number of used pairs', 'Absorption name'], - extname='ATTRI') + comment=["Number of pairs", "Number of used pairs", "Absorption name"], + extname="ATTRI", + ) dmat_name = "DM_" if blinding != "none": @@ -572,37 +620,40 @@ def main(cmdargs=None): out_comment = [] out_units = [] for index, name in enumerate(names): - out_names += ['RP_' + name] + out_names += ["RP_" + name] out_list += [r_par_all[index]] - out_comment += ['R-parallel'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-parallel"] + out_units += ["h^-1 Mpc"] - out_names += ['RT_' + name] + out_names += ["RT_" + name] out_list += [r_trans_all[index]] - out_comment += ['R-transverse'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-transverse"] + out_units += ["h^-1 Mpc"] - out_names += ['Z_' + name] + out_names += ["Z_" + name] out_list += [z_all[index]] - out_comment += ['Redshift'] - out_units += [''] + out_comment += ["Redshift"] + out_units += [""] out_names += [dmat_name + name] out_list += [dmat_all[index]] - out_comment += ['Distortion matrix'] - out_units += [''] + out_comment += ["Distortion matrix"] + out_units += [""] - out_names += ['WDM_' + name] + out_names += ["WDM_" + name] out_list += [weights_dmat_all[index]] - out_comment += ['Sum of weight'] - out_units += [''] - - results.write(out_list, - names=out_names, - comment=out_comment, - units=out_units, - extname='MDMAT') + out_comment += ["Sum of weight"] + out_units += [""] + + results.write( + out_list, names=out_names, comment=out_comment, units=out_units, extname="MDMAT" + ) results.close() t3 = time.time() - userprint(f'picca_metal_dmat.py - Time total : {(t3-t0)/60:.3f} minutes') + userprint(f"picca_metal_dmat.py - Time total : {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_metal_xdmat.py b/py/picca/bin/picca_metal_xdmat.py old mode 100644 new mode 100755 index b63885e06..9ff5cbe41 --- a/py/picca/bin/picca_metal_xdmat.py +++ b/py/picca/bin/picca_metal_xdmat.py @@ -1,19 +1,21 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the distortion matrix of the cross-correlation delta x object for a list of IGM absorption. This module follow the procedure described in sections 4.3 of du Mas des Bourboux et al. 2020 (In prep) to compute the distortion matrix """ -import time import argparse -from functools import partial import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value -import numpy as np +import sys +import time +from functools import partial +from multiprocessing import Lock, Value, cpu_count + import fitsio +import numpy as np -from picca import constants, xcf, io, utils +from picca import constants, io, utils, xcf from picca.utils import userprint @@ -42,233 +44,271 @@ def calc_metal_dmat(abs_igm, healpixs): def main(cmdargs=None): """Compute the distortion matrix of the cross-correlation delta x object for - a list of IGM absorption.""" + a list of IGM absorption.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the distortion matrix of the cross-correlation ' - 'delta x object for a list of IGM absorption.')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--drq', - type=str, - default=None, - required=True, - help='Catalog of objects in DRQ format') - - parser.add_argument( - '--mode', - type=str, - default='sdss', - choices=['sdss','desi','desi_mocks','desi_healpix'], - required=False, - help='type of catalog supplied, default sdss') - - parser.add_argument('--rp-min', - type=float, - default=-200., - required=False, - help='Min r-parallel [h^-1 Mpc]') - - parser.add_argument('--rp-max', - type=float, - default=200., - required=False, - help='Max r-parallel [h^-1 Mpc]') - - parser.add_argument('--rt-max', - type=float, - default=200., - required=False, - help='Max r-transverse [h^-1 Mpc]') - - parser.add_argument('--np', - type=int, - default=100, - required=False, - help='Number of r-parallel bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of r-transverse bins') - - parser.add_argument( - '--coef-binning-model', + description=( + "Compute the distortion matrix of the cross-correlation " + "delta x object for a list of IGM absorption." + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--drq", + type=str, + default=None, + required=True, + help="Catalog of objects in DRQ format", + ) + + parser.add_argument( + "--mode", + type=str, + default="sdss", + choices=["sdss", "desi", "desi_mocks", "desi_healpix"], + required=False, + help="type of catalog supplied, default sdss", + ) + + parser.add_argument( + "--rp-min", + type=float, + default=-200.0, + required=False, + help="Min r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rp-max", + type=float, + default=200.0, + required=False, + help="Max r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rt-max", + type=float, + default=200.0, + required=False, + help="Max r-transverse [h^-1 Mpc]", + ) + + parser.add_argument( + "--np", type=int, default=100, required=False, help="Number of r-parallel bins" + ) + + parser.add_argument( + "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" + ) + + parser.add_argument( + "--coef-binning-model", type=int, default=1, required=False, - help=('Coefficient multiplying np and nt to get finner binning for the ' - 'model')) + help=( + "Coefficient multiplying np and nt to get finner binning for the " "model" + ), + ) - parser.add_argument('--z-min-obj', - type=float, - default=0, - required=False, - help='Min redshift for object field') + parser.add_argument( + "--z-min-obj", + type=float, + default=0, + required=False, + help="Min redshift for object field", + ) - parser.add_argument('--z-max-obj', - type=float, - default=10, - required=False, - help='Max redshift for object field') + parser.add_argument( + "--z-max-obj", + type=float, + default=10, + required=False, + help="Max redshift for object field", + ) parser.add_argument( - '--z-cut-min', + "--z-cut-min", type=float, - default=0., + default=0.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--z-cut-max', + "--z-cut-max", type=float, - default=10., + default=10.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) parser.add_argument( - '--z-min-sources', + "--z-min-sources", type=float, - default=0., + default=0.0, required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--z-max-sources', + "--z-max-sources", type=float, - default=10., + default=10.0, required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--lambda-abs', + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) - parser.add_argument('--obj-name', - type=str, - default='QSO', - required=False, - help='Name of the object tracer') + parser.add_argument( + "--obj-name", + type=str, + default="QSO", + required=False, + help="Name of the object tracer", + ) parser.add_argument( - '--abs-igm', + "--abs-igm", type=str, default=None, required=False, - nargs='*', - help='List of names of metal absorption in picca.constants') + nargs="*", + help="List of names of metal absorption in picca.constants", + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol-del', + "--z-evol-del", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol-obj', + "--z-evol-obj", type=float, default=1.44, required=False, - help='Exponent of the redshift evolution of the object field') + help="Exponent of the redshift evolution of the object field", + ) parser.add_argument( - '--metal-alpha', + "--metal-alpha", type=float, - default=1., + default=1.0, required=False, - help='Exponent of the redshift evolution of the metal delta field') + help="Exponent of the redshift evolution of the metal delta field", + ) parser.add_argument( - '--fid-Om', + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--rej', + "--rej", type=float, - default=1., + default=1.0, required=False, - help=('Fraction of rejected object-forests pairs: -1=no rejection, ' - '1=all rejection')) - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') - - parser.add_argument('--rebin-factor', - type=int, - default=None, - required=False, - help='Rebin factor for deltas. If not None, deltas will ' - 'be rebinned by that factor') + help=( + "Fraction of rejected object-forests pairs: -1=no rejection, " + "1=all rejection" + ), + ) - args = parser.parse_args(cmdargs) + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) + + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, + required=False, + help="Maximum number of spectra to read", + ) + + parser.add_argument( + "--rebin-factor", + type=int, + default=None, + required=False, + help="Rebin factor for deltas. If not None, deltas will " + "be rebinned by that factor", + ) + args = parser.parse_args(cmdargs) if args.nproc is None: args.nproc = cpu_count() // 2 @@ -297,26 +337,30 @@ def main(cmdargs=None): blinding = io.read_blinding(args.in_dir) # load fiducial cosmology - cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl,) + cosmo = constants.Cosmo( + Om=args.fid_Om, + Or=args.fid_Or, + Ok=args.fid_Ok, + wl=args.fid_wl, + ) xcf.cosmo = cosmo t0 = time.time() # read deltas - data, num_data, z_min, z_max = io.read_deltas(args.in_dir, - args.nside, - xcf.lambda_abs, - args.z_evol_del, - args.z_ref, - cosmo, - max_num_spec=args.nspec, - nproc=args.nproc, - rebin_factor=args.rebin_factor, - z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + data, num_data, z_min, z_max = io.read_deltas( + args.in_dir, + args.nside, + xcf.lambda_abs, + args.z_evol_del, + args.z_ref, + cosmo, + max_num_spec=args.nspec, + nproc=args.nproc, + rebin_factor=args.rebin_factor, + z_min_qso=args.z_min_sources, + z_max_qso=args.z_max_sources, + ) xcf.data = data xcf.num_data = num_data @@ -324,28 +368,35 @@ def main(cmdargs=None): ### Find the redshift range if args.z_min_obj is None: r_comov_min = cosmo.get_r_comov(z_min) - r_comov_min = max(0., r_comov_min + xcf.r_par_min) + r_comov_min = max(0.0, r_comov_min + xcf.r_par_min) args.z_min_obj = cosmo.distance_to_redshift(r_comov_min) userprint("z_min_obj = {}".format(args.z_min_obj), end="") if args.z_max_obj is None: r_comov_max = cosmo.get_r_comov(z_max) - r_comov_max = max(0., r_comov_max + xcf.r_par_max) + r_comov_max = max(0.0, r_comov_max + xcf.r_par_max) args.z_max_obj = cosmo.distance_to_redshift(r_comov_max) userprint("z_max_obj = {}".format(args.z_max_obj), end="") # read objets - objs, z_min2 = io.read_objects(args.drq, args.nside, args.z_min_obj, - args.z_max_obj, args.z_evol_obj, args.z_ref, - cosmo, mode=args.mode) + objs, z_min2 = io.read_objects( + args.drq, + args.nside, + args.z_min_obj, + args.z_max_obj, + args.z_evol_obj, + args.z_ref, + cosmo, + mode=args.mode, + ) xcf.objs = objs # compute maximum angular separation xcf.ang_max = utils.compute_ang_max(cosmo, xcf.r_trans_max, z_min, z_min2) t1 = time.time() - userprint(f'picca_metal_xdmat.py - Time reading data: {(t1-t0)/60:.3f} minutes') + userprint(f"picca_metal_xdmat.py - Time reading data: {(t1-t0)/60:.3f} minutes") - xcf.counter = Value('i', 0) + xcf.counter = Value("i", 0) xcf.lock = Lock() cpu_data = {} for index, healpix in enumerate(sorted(list(data.keys()))): @@ -371,10 +422,9 @@ def main(cmdargs=None): userprint("") if args.nproc > 1: - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=args.nproc) - dmat_data = pool.map(calc_metal_dmat_wrapper, - sorted(cpu_data.values())) + dmat_data = pool.map(calc_metal_dmat_wrapper, sorted(cpu_data.values())) pool.close() elif args.nproc == 1: dmat_data = map(calc_metal_dmat_wrapper, sorted(cpu_data.values())) @@ -409,89 +459,89 @@ def main(cmdargs=None): num_pairs_used_all.append(num_pairs_used) t2 = time.time() - userprint(f'picca_metal_xdmat.py - Time computing all metal matrix: {(t2-t1)/60:.3f} minutes') + userprint( + f"picca_metal_xdmat.py - Time computing all metal matrix: {(t2-t1)/60:.3f} minutes" + ) # save the results - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = [ { - 'name': 'RPMIN', - 'value': xcf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' + "name": "RPMIN", + "value": xcf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": xcf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", + }, + { + "name": "RTMAX", + "value": xcf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", + }, + { + "name": "NP", + "value": xcf.num_bins_r_par, + "comment": "Number of bins in r-parallel", + }, + { + "name": "NT", + "value": xcf.num_bins_r_trans, + "comment": "Number of bins in r-transverse", }, { - 'name': 'RPMAX', - 'value': xcf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' + "name": "COEFMOD", + "value": args.coef_binning_model, + "comment": "Coefficient for model binning", }, { - 'name': 'RTMAX', - 'value': xcf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' + "name": "ZCUTMIN", + "value": xcf.z_cut_min, + "comment": "Minimum redshift of pairs", }, { - 'name': 'NP', - 'value': xcf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' + "name": "ZCUTMAX", + "value": xcf.z_cut_max, + "comment": "Maximum redshift of pairs", }, + {"name": "REJ", "value": xcf.reject, "comment": "Rejection factor"}, { - 'name': 'NT', - 'value': xcf.num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'COEFMOD', - 'value': args.coef_binning_model, - 'comment': 'Coefficient for model binning' + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'ZCUTMIN', - 'value': xcf.z_cut_min, - 'comment': 'Minimum redshift of pairs' + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'ZCUTMAX', - 'value': xcf.z_cut_max, - 'comment': 'Maximum redshift of pairs' + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", }, { - 'name': 'REJ', - 'value': xcf.reject, - 'comment': 'Rejection factor' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - } - ] + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + ] len_names = np.array([len(name) for name in names]).max() - names = np.array(names, dtype='S' + str(len_names)) + names = np.array(names, dtype="S" + str(len_names)) results.write( - [ - np.array(num_pairs_all), - np.array(num_pairs_used_all), - np.array(names) - ], - names=['NPALL', 'NPUSED', 'ABS_IGM'], + [np.array(num_pairs_all), np.array(num_pairs_used_all), np.array(names)], + names=["NPALL", "NPUSED", "ABS_IGM"], header=header, - comment=['Number of pairs', 'Number of used pairs', 'Absorption name'], - extname='ATTRI') + comment=["Number of pairs", "Number of used pairs", "Absorption name"], + extname="ATTRI", + ) dmat_name = "DM_" if blinding != "none": @@ -502,37 +552,40 @@ def main(cmdargs=None): out_comment = [] out_units = [] for index, name in enumerate(names): - out_names += ['RP_' + args.obj_name + '_' + name] + out_names += ["RP_" + args.obj_name + "_" + name] out_list += [r_par_all[index]] - out_comment += ['R-parallel'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-parallel"] + out_units += ["h^-1 Mpc"] - out_names += ['RT_' + args.obj_name + '_' + name] + out_names += ["RT_" + args.obj_name + "_" + name] out_list += [r_trans_all[index]] - out_comment += ['R-transverse'] - out_units += ['h^-1 Mpc'] + out_comment += ["R-transverse"] + out_units += ["h^-1 Mpc"] - out_names += ['Z_' + args.obj_name + '_' + name] + out_names += ["Z_" + args.obj_name + "_" + name] out_list += [z_all[index]] - out_comment += ['Redshift'] - out_units += [''] + out_comment += ["Redshift"] + out_units += [""] - out_names += [dmat_name + args.obj_name + '_' + name] + out_names += [dmat_name + args.obj_name + "_" + name] out_list += [dmat_all[index]] - out_comment += ['Distortion matrix'] - out_units += [''] + out_comment += ["Distortion matrix"] + out_units += [""] - out_names += ['WDM_' + args.obj_name + '_' + name] + out_names += ["WDM_" + args.obj_name + "_" + name] out_list += [weights_dmat_all[index]] - out_comment += ['Sum of weight'] - out_units += [''] - - results.write(out_list, - names=out_names, - comment=out_comment, - units=out_units, - extname='MDMAT') + out_comment += ["Sum of weight"] + out_units += [""] + + results.write( + out_list, names=out_names, comment=out_comment, units=out_units, extname="MDMAT" + ) results.close() t3 = time.time() - userprint(f'picca_metal_xdmat.py - Time total: {(t3-t0)/60:.3f} minutes') + userprint(f"picca_metal_xdmat.py - Time total: {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_nersc_submit.py b/py/picca/bin/picca_nersc_submit.py old mode 100644 new mode 100755 index 3dc492589..f1d2d2f34 --- a/py/picca/bin/picca_nersc_submit.py +++ b/py/picca/bin/picca_nersc_submit.py @@ -1,13 +1,13 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Writes scripts to send the picca BAO analysis Once ran, run `./submit.sh` in your terminal. This script works on nersc::cori not on nersc::edison. """ -import sys +import argparse import os +import sys from os.path import basename, dirname, stat -import argparse class Batch: @@ -83,16 +83,18 @@ def get_header(time, name, email=None, queue="regular", account="desi"): return header -def picca_deltas_script(batch, - time, - in_dir, - out_dir, - drq_filename, - email=None, - debug=False, - mode="desi", - lambda_rest_frame_min=None, - lambda_rest_frame_max=None): +def picca_deltas_script( + batch, + time, + in_dir, + out_dir, + drq_filename, + email=None, + debug=False, + mode="desi", + lambda_rest_frame_min=None, + lambda_rest_frame_max=None, +): """Writes a .batch file to submit the picca_deltas.py script Args: @@ -122,11 +124,13 @@ def picca_deltas_script(batch, if mode == "eboss": mode = "spplate" header = get_header(time, name="picca_deltas", email=email) - header += ("srun -n 1 -c 64 picca_deltas.py " + - "--in-dir {} --drq {} ".format(in_dir, drq_filename) + - "--out-dir {}/deltas/ --mode {} ".format(out_dir, mode) + - "--iter-out-prefix {}/iter ".format(out_dir) + - "--log {}/input.log".format(out_dir)) + header += ( + "srun -n 1 -c 64 picca_deltas.py " + + "--in-dir {} --drq {} ".format(in_dir, drq_filename) + + "--out-dir {}/deltas/ --mode {} ".format(out_dir, mode) + + "--iter-out-prefix {}/iter ".format(out_dir) + + "--log {}/input.log".format(out_dir) + ) if lambda_rest_frame_min is not None: header += " --lambda-rest-min {}".format(lambda_rest_frame_min) if lambda_rest_frame_max is not None: @@ -141,14 +145,9 @@ def picca_deltas_script(batch, file.close() -def cf_script(batch, - time, - z_intervals, - out_dir, - email=None, - fid_om=None, - fid_pk=None, - fid_or=None): +def cf_script( + batch, time, z_intervals, out_dir, email=None, fid_om=None, fid_pk=None, fid_or=None +): """ Writes the .batch files to submit the picca_cf.py and the picca_export.py scripts and adds them to the batch.cf and batch.export lists @@ -180,13 +179,18 @@ def cf_script(batch, out_dir + "/cf_z_{}_{}.fits".format(z_min, z_max), out_dir + "/dmat_z_{}_{}.fits".format(z_min, z_max), out_dir + "/cf_z_{}_{}-exp.fits".format(z_min, z_max), - fid_pk=fid_pk) + fid_pk=fid_pk, + ) header = get_header(time, name=out, email=email) - srun = (header + "srun -n 1 -c 64 " - "picca_cf.py --in-dir {}/deltas/ ".format(out_dir) + - "--z-cut-min {} --z-cut-max {} ".format(z_min, z_max) + - "--out {}/{} --nproc 32 ".format(out_dir, out) + - "--fid-Om {} --fid-Or {} \n".format(fid_om, fid_or)) + srun = header + "srun -n 1 -c 64 " "picca_cf.py --in-dir {}/deltas/ ".format( + out_dir + ) + "--z-cut-min {} --z-cut-max {} ".format( + z_min, z_max + ) + "--out {}/{} --nproc 32 ".format( + out_dir, out + ) + "--fid-Om {} --fid-Or {} \n".format( + fid_om, fid_or + ) batch_filename = out_dir + "/" + out.replace(".fits", ".batch") batch.cf_script_list.append(basename(batch_filename)) @@ -197,14 +201,9 @@ def cf_script(batch, file.close() -def dmat_script(batch, - time, - z_intervals, - out_dir, - email=None, - reject=0.99, - fid_om=None, - fid_or=None): +def dmat_script( + batch, time, z_intervals, out_dir, email=None, reject=0.99, fid_om=None, fid_or=None +): """ Writes the .batch files to submit the picca_dmat.py script and adds them to the batch.dmat script @@ -231,11 +230,14 @@ def dmat_script(batch, z_min, z_max = z_interval.split(":") out = "dmat_z_{}_{}.fits".format(z_min, z_max) header = get_header(time, name=out, email=email) - srun = (header + "srun -n 1 -c 64 " + - "picca_dmat.py --in-dir {}/deltas/ ".format(out_dir) + - "--z-cut-min {} --z-cut-max {} ".format(z_min, z_max) + - "--out {}/{} --rej {} ".format(out_dir, out, reject) + - "--nproc 32 --fid-Om {} --fid-Or {}\n".format(fid_om, fid_or)) + srun = ( + header + + "srun -n 1 -c 64 " + + "picca_dmat.py --in-dir {}/deltas/ ".format(out_dir) + + "--z-cut-min {} --z-cut-max {} ".format(z_min, z_max) + + "--out {}/{} --rej {} ".format(out_dir, out, reject) + + "--nproc 32 --fid-Om {} --fid-Or {}\n".format(fid_om, fid_or) + ) batch_filename = out_dir + "/" + out.replace(".fits", ".batch") batch.dmat_script_list.append(basename(batch_filename)) @@ -244,15 +246,17 @@ def dmat_script(batch, file.close() -def xcf_script(batch, - time, - drq_filename, - z_intervals, - out_dir, - email=None, - fid_om=None, - fid_pk=None, - fid_or=None): +def xcf_script( + batch, + time, + drq_filename, + z_intervals, + out_dir, + email=None, + fid_om=None, + fid_pk=None, + fid_or=None, +): """ Writes the .batch files to submit the picca_xcf.py script and the picca_export.py scripts @@ -288,13 +292,17 @@ def xcf_script(batch, out_dir + "/xcf_z_{}_{}.fits".format(z_min, z_max), out_dir + "/xdmat_z_{}_{}.fits".format(z_min, z_max), out_dir + "/xcf_z_{}_{}-exp.fits".format(z_min, z_max), - fid_pk=fid_pk) - srun = (header + "srun -n 1 -c 64 picca_xcf.py " + - "--drq {} --in-dir {}/deltas/ ".format(drq_filename, out_dir) + - "--z-evol-obj 1.44 " + - "--z-cut-min {} --z-cut-max {} ".format(z_min, z_max) + - "--out {}/{} --nproc 32 ".format(out_dir, out) + - "--fid-Om {} --fid-Or {}\n".format(fid_om, fid_or)) + fid_pk=fid_pk, + ) + srun = ( + header + + "srun -n 1 -c 64 picca_xcf.py " + + "--drq {} --in-dir {}/deltas/ ".format(drq_filename, out_dir) + + "--z-evol-obj 1.44 " + + "--z-cut-min {} --z-cut-max {} ".format(z_min, z_max) + + "--out {}/{} --nproc 32 ".format(out_dir, out) + + "--fid-Om {} --fid-Or {}\n".format(fid_om, fid_or) + ) batch_filename = out_dir + "/" + out.replace(".fits", ".batch") batch.xcf_script_list.append(basename(batch_filename)) batch.xexport_script_list.append(basename(exp_batch)) @@ -304,15 +312,17 @@ def xcf_script(batch, file.close() -def xdmat_script(batch, - time, - drq_filename, - z_intervals, - out_dir, - email=None, - reject=0.95, - fid_om=None, - fid_or=None): +def xdmat_script( + batch, + time, + drq_filename, + z_intervals, + out_dir, + email=None, + reject=0.95, + fid_om=None, + fid_or=None, +): """ Writes the .batch files to submit the picca_xdmat.py script and adds if to the b.xdmat list @@ -342,13 +352,16 @@ def xdmat_script(batch, z_min, z_max = z_interval.split(":") out = "xdmat_z_{}_{}.fits".format(z_min, z_max) header = get_header(time, name=out, email=email) - srun = (header + "srun -n 1 -c 64 picca_xdmat.py " + - "--drq {} --in-dir {}/deltas/ ".format(drq_filename, out_dir) + - "--z-evol-obj 1.44 " + - "--z-cut-min {} --z-cut-max {} ".format(z_min, z_max) + - "--out {}/{} ".format(out_dir, out) + - "--rej {} --nproc 32 ".format(reject) + - "--fid-Om {} --fid-Or {}\n".format(fid_om, fid_or)) + srun = ( + header + + "srun -n 1 -c 64 picca_xdmat.py " + + "--drq {} --in-dir {}/deltas/ ".format(drq_filename, out_dir) + + "--z-evol-obj 1.44 " + + "--z-cut-min {} --z-cut-max {} ".format(z_min, z_max) + + "--out {}/{} ".format(out_dir, out) + + "--rej {} --nproc 32 ".format(reject) + + "--fid-Om {} --fid-Or {}\n".format(fid_om, fid_or) + ) batch_filename = out_dir + "/" + out.replace(".fits", ".batch") batch.xdmat_script_list.append(basename(batch_filename)) @@ -379,9 +392,12 @@ def export_script(time, cf_file, dmat_file, out, fid_pk): picca_fitter2.py """ header = get_header(time, name=basename(out), queue="regular") - srun = (header + "srun -n 1 -c 64 picca_export.py " + - "--data {} --dmat {} ".format(cf_file, dmat_file) + - "--out {}\n".format(out)) + srun = ( + header + + "srun -n 1 -c 64 picca_export.py " + + "--data {} --dmat {} ".format(cf_file, dmat_file) + + "--out {}\n".format(out) + ) chi2_ini = do_ini(dirname(out), basename(out), fid_pk) srun += "srun -n 1 -c 64 picca_fitter2.py {}\n".format(chi2_ini) batch_filename = out.replace(".fits", ".batch") @@ -417,7 +433,7 @@ def do_ini(out_dir, cf_file, fid_pk): else: file.write("tracer2 = LYA\n") file.write("tracer2-type = continuous\n") - file.write("filename = {}\n".format(out_dir + '/' + cf_file)) + file.write("filename = {}\n".format(out_dir + "/" + cf_file)) file.write("ell-max = 6\n") file.write("[cuts]\n") @@ -461,10 +477,16 @@ def do_ini(out_dir, cf_file, fid_pk): file.write("sigmaNL_par = 6.36984 0.1 None None fixed\n") file.write("sigmaNL_per = 3.24 0.1 None None fixed\n") - file.write(("par binsize {} = 4. 0.4 None None " - "free\n").format(cf_file.replace(".fits", ""))) - file.write(("per binsize {} = 4. 0.4 None None " - "free\n").format(cf_file.replace(".fits", ""))) + file.write( + ("par binsize {} = 4. 0.4 None None " "free\n").format( + cf_file.replace(".fits", "") + ) + ) + file.write( + ("per binsize {} = 4. 0.4 None None " "free\n").format( + cf_file.replace(".fits", "") + ) + ) file.write("bao_amp = 1. 0.1 None None fixed\n") if "xcf" in cf_file: @@ -476,8 +498,9 @@ def do_ini(out_dir, cf_file, fid_pk): file = open(chi2_ini, "w") file.write("[data sets]\n") file.write("zeff = 2.310\n") - file.write("ini files = {}\n".format(out_dir + "/" + - cf_file.replace(".fits", ".ini"))) + file.write( + "ini files = {}\n".format(out_dir + "/" + cf_file.replace(".fits", ".ini")) + ) file.write("[fiducial]\n") file.write("filename = {}\n".format(fid_pk)) @@ -486,8 +509,9 @@ def do_ini(out_dir, cf_file, fid_pk): file.write("level = 0\n") file.write("[output]\n") - file.write("filename = {}\n".format(out_dir + "/" + - cf_file.replace(".fits", ".h5"))) + file.write( + "filename = {}\n".format(out_dir + "/" + cf_file.replace(".fits", ".h5")) + ) file.write("[cosmo-fit type]\n") file.write("cosmo fit func = ap_at\n") @@ -507,62 +531,84 @@ def submit(batch): file = open(out_name, "w") file.write("#!/bin/bash\n") if batch.picca_deltas_script_filename is not None: - file.write(("picca_deltas=$(sbatch --parsable " - "{})\n").format(batch.picca_deltas_script_filename)) + file.write( + ("picca_deltas=$(sbatch --parsable " "{})\n").format( + batch.picca_deltas_script_filename + ) + ) file.write('echo "picca_deltas: "$picca_deltas\n') - for cf_batch, dmat_batch, exp_batch in zip(batch.cf_script_list, - batch.dmat_script_list, - batch.export_script_list): + for cf_batch, dmat_batch, exp_batch in zip( + batch.cf_script_list, batch.dmat_script_list, batch.export_script_list + ): var_cf = cf_batch.replace(".batch", "").replace(".", "_") var_dmat = dmat_batch.replace(".batch", "").replace(".", "_") if batch.picca_deltas_script_filename is not None: - file.write(("{}=$(sbatch --parsable " - "--dependency=afterok:$picca_deltas " - "{})\n").format(var_cf, cf_batch)) + file.write( + ( + "{}=$(sbatch --parsable " + "--dependency=afterok:$picca_deltas " + "{})\n" + ).format(var_cf, cf_batch) + ) file.write('echo "{0}: "${0} \n'.format(var_cf)) - file.write(("{}=$(sbatch --parsable " - "--dependency=afterok:$picca_deltas " - "{})\n").format(var_dmat, dmat_batch)) + file.write( + ( + "{}=$(sbatch --parsable " + "--dependency=afterok:$picca_deltas " + "{})\n" + ).format(var_dmat, dmat_batch) + ) file.write('echo "{0}: "${0} \n'.format(var_dmat)) else: file.write("{}=$(sbatch --parsable {})\n".format(var_cf, cf_batch)) file.write('echo "{0}: "${0} \n'.format(var_cf)) - file.write("{}=$(sbatch --parsable {})\n".format( - var_dmat, dmat_batch)) + file.write("{}=$(sbatch --parsable {})\n".format(var_dmat, dmat_batch)) file.write('echo "{0}: "${0} \n'.format(var_dmat)) - var_exp = exp_batch.replace(".batch", - "").replace(".", "_").replace("-", "_") - file.write(("{}=$(sbatch --parsable " - "--dependency=afterok:${},afterok:${} " - "{})\n").format(var_exp, var_cf, var_dmat, exp_batch)) + var_exp = exp_batch.replace(".batch", "").replace(".", "_").replace("-", "_") + file.write( + ( + "{}=$(sbatch --parsable " + "--dependency=afterok:${},afterok:${} " + "{})\n" + ).format(var_exp, var_cf, var_dmat, exp_batch) + ) file.write('echo "{0}: "${0} \n'.format(var_exp)) - for xcf_batch, xdmat_batch, xexp_batch in zip(batch.xcf_script_list, - batch.xdmat_script_list, - batch.xexport_script_list): + for xcf_batch, xdmat_batch, xexp_batch in zip( + batch.xcf_script_list, batch.xdmat_script_list, batch.xexport_script_list + ): var_xcf = xcf_batch.replace(".batch", "").replace(".", "_") var_xdmat = xdmat_batch.replace(".batch", "").replace(".", "_") if batch.picca_deltas_script_filename is not None: - file.write(("{}=$(sbatch --parsable " - "--dependency=afterok:$picca_deltas " - "{})\n").format(var_xcf, xcf_batch)) + file.write( + ( + "{}=$(sbatch --parsable " + "--dependency=afterok:$picca_deltas " + "{})\n" + ).format(var_xcf, xcf_batch) + ) file.write('echo "{0}: "${0} \n'.format(var_xcf)) - file.write(("{}=$(sbatch --parsable " - "--dependency=afterok:$picca_deltas " - "{})\n").format(var_xdmat, xdmat_batch)) + file.write( + ( + "{}=$(sbatch --parsable " + "--dependency=afterok:$picca_deltas " + "{})\n" + ).format(var_xdmat, xdmat_batch) + ) file.write('echo "{0}: "${0} \n'.format(var_xdmat)) else: - file.write(("{}=$(sbatch --parsable " - "{})\n").format(var_xcf, xcf_batch)) + file.write(("{}=$(sbatch --parsable " "{})\n").format(var_xcf, xcf_batch)) file.write('echo "{0}: "${0} \n'.format(var_xcf)) - file.write( - ("{}=$(sbatch --parsable {})\n").format(var_xdmat, xdmat_batch)) + file.write(("{}=$(sbatch --parsable {})\n").format(var_xdmat, xdmat_batch)) file.write('echo "{0}: "${0} \n'.format(var_xdmat)) - var_xexp = xexp_batch.replace(".batch", - "").replace(".", "_").replace("-", "_") - file.write(("{}=$(sbatch --parsable " - "--dependency=afterok:${},afterok:${} " - "{})\n").format(var_xexp, var_xcf, var_xdmat, xexp_batch)) + var_xexp = xexp_batch.replace(".batch", "").replace(".", "_").replace("-", "_") + file.write( + ( + "{}=$(sbatch --parsable " + "--dependency=afterok:${},afterok:${} " + "{})\n" + ).format(var_xexp, var_xcf, var_xdmat, xexp_batch) + ) file.write('echo "{0}: "${0} \n'.format(var_xexp)) file.close() @@ -577,73 +623,78 @@ def main(cmdargs): """ parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Writes scripts to send the picca BAO analysis. Once ran, ' - 'run `./submit.sh` in your terminal. This script works on ' - 'nersc::cori not on nersc::edison.')) - - parser.add_argument("--out-dir", - type=str, - default=None, - required=True, - help="Output directory") - - parser.add_argument("--drq", - type=str, - default=None, - required=True, - help="Absolute path to drq file") + description=( + "Writes scripts to send the picca BAO analysis. Once ran, " + "run `./submit.sh` in your terminal. This script works on " + "nersc::cori not on nersc::edison." + ), + ) + + parser.add_argument( + "--out-dir", type=str, default=None, required=True, help="Output directory" + ) + + parser.add_argument( + "--drq", type=str, default=None, required=True, help="Absolute path to drq file" + ) parser.add_argument( "--in-dir", type=str, default=None, required=True, - help=("Absolute path to spectra-NSIDE directory (including " - "spectra-NSIDE)")) - - parser.add_argument("--email", - type=str, - default=None, - required=False, - help="Your email address (optional)") - - parser.add_argument("--to-do", - type=str, - nargs="*", - default=["cf", "xcf"], - required=False, - help="What to do") - - parser.add_argument("--fid-Om", - type=float, - default=0.3147, - required=False, - help="Fiducial Om") - - parser.add_argument("--fid-Or", - type=float, - default=0., - required=False, - help="Fiducial Or") - - parser.add_argument("--fid-Pk", - type=str, - default="PlanckDR12/PlanckDR12.fits", - required=False, - help="Fiducial Pk") - - parser.add_argument("--zint", - type=str, - nargs="*", - default=['0:2.35', '2.35:2.65', '2.65:3.05', '3.05:10'], - required=False, - help="Redshifts intervals") - - parser.add_argument("--mode", - type=str, - default="desi", - required=False, - help="Use eboss or desi data") + help=("Absolute path to spectra-NSIDE directory (including " "spectra-NSIDE)"), + ) + + parser.add_argument( + "--email", + type=str, + default=None, + required=False, + help="Your email address (optional)", + ) + + parser.add_argument( + "--to-do", + type=str, + nargs="*", + default=["cf", "xcf"], + required=False, + help="What to do", + ) + + parser.add_argument( + "--fid-Om", type=float, default=0.3147, required=False, help="Fiducial Om" + ) + + parser.add_argument( + "--fid-Or", type=float, default=0.0, required=False, help="Fiducial Or" + ) + + parser.add_argument( + "--fid-Pk", + type=str, + default="PlanckDR12/PlanckDR12.fits", + required=False, + help="Fiducial Pk", + ) + + parser.add_argument( + "--zint", + type=str, + nargs="*", + default=["0:2.35", "2.35:2.65", "2.65:3.05", "3.05:10"], + required=False, + help="Redshifts intervals", + ) + + parser.add_argument( + "--mode", + type=str, + default="desi", + required=False, + help="Use eboss or desi data", + ) parser.add_argument("--debug", action="store_true", default=False) @@ -651,19 +702,24 @@ def main(cmdargs): "--no-deltas", action="store_true", default=False, - help="Do not run picca_deltas (e.g. because they were already run)") + help="Do not run picca_deltas (e.g. because they were already run)", + ) - parser.add_argument('--lambda-rest-min', - type=float, - default=None, - required=False, - help='Lower limit on rest frame wavelength [Angstrom]') + parser.add_argument( + "--lambda-rest-min", + type=float, + default=None, + required=False, + help="Lower limit on rest frame wavelength [Angstrom]", + ) - parser.add_argument('--lambda-rest-max', - type=float, - default=None, - required=False, - help='Upper limit on rest frame wavelength [Angstrom]') + parser.add_argument( + "--lambda-rest-max", + type=float, + default=None, + required=False, + help="Upper limit on rest frame wavelength [Angstrom]", + ) args = parser.parse_args(cmdargs) @@ -680,70 +736,80 @@ def main(cmdargs): time = "03:30:00" if args.debug: time = time_debug - cf_script(batch, - time, - args.zint, - args.out_dir, - email=args.email, - fid_om=args.fid_om, - fid_pk=args.fid_Pk, - fid_or=args.fid_Or) + cf_script( + batch, + time, + args.zint, + args.out_dir, + email=args.email, + fid_om=args.fid_om, + fid_pk=args.fid_Pk, + fid_or=args.fid_Or, + ) time = "02:00:00" if args.debug: time = time_debug - dmat_script(batch, - time, - args.zint, - args.out_dir, - email=args.email, - fid_om=args.fid_om, - fid_or=args.fid_Or) + dmat_script( + batch, + time, + args.zint, + args.out_dir, + email=args.email, + fid_om=args.fid_om, + fid_or=args.fid_Or, + ) if "xcf" in args.to_do: time = "01:30:00" if args.debug: time = time_debug - xcf_script(batch, - time, - args.drq, - args.zint, - args.out_dir, - email=args.email, - fid_om=args.fid_om, - fid_pk=args.fid_Pk, - fid_or=args.fid_Or) + xcf_script( + batch, + time, + args.drq, + args.zint, + args.out_dir, + email=args.email, + fid_om=args.fid_om, + fid_pk=args.fid_Pk, + fid_or=args.fid_Or, + ) time = "03:00:00" if args.debug: time = time_debug - xdmat_script(batch, - time, - args.drq, - args.zint, - args.out_dir, - email=args.email, - fid_om=args.fid_om, - fid_or=args.fid_Or) + xdmat_script( + batch, + time, + args.drq, + args.zint, + args.out_dir, + email=args.email, + fid_om=args.fid_om, + fid_or=args.fid_Or, + ) time = "02:00:00" if args.debug: time = time_debug if not args.no_deltas: - picca_deltas_script(batch, - time, - args.in_dir, - args.out_dir, - args.drq, - email=args.email, - debug=args.debug, - mode=args.mode, - lambda_rest_frame_min=args.lambda_rest_min, - lambda_rest_frame_max=args.lambda_rest_max) + picca_deltas_script( + batch, + time, + args.in_dir, + args.out_dir, + args.drq, + email=args.email, + debug=args.debug, + mode=args.mode, + lambda_rest_frame_min=args.lambda_rest_min, + lambda_rest_frame_max=args.lambda_rest_max, + ) submit(batch) if __name__ == "__main__": - cmdargs=sys.argv[1:] + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_pk2fits.py b/py/picca/bin/picca_pk2fits.py old mode 100644 new mode 100755 index 2316c0d18..2fc12b51d --- a/py/picca/bin/picca_pk2fits.py +++ b/py/picca/bin/picca_pk2fits.py @@ -1,65 +1,98 @@ -#!/usr/bin/env python -import sys -from astropy.io import fits -import numpy as np +#!/usr/bin/env python3 import argparse +import sys -def main(cmdargs): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument('--prefix-pk', type=str, default=None, required=True, - help='Prefix to pk file') - - parser.add_argument('--out', type=str, default=None, required=True, - help='Output file name') - - parser.add_argument('--zref', type=float, default=None, required=True, - help='Reference redshift') - - parser.add_argument('--Om', type=float, default=None, required=True, - help='Matter density parameter') - - parser.add_argument('--Ok', type=float, default=None, required=True, - help='Curvature density parameter') - - parser.add_argument('--w', type=float, default=None, required=True, - help='Dark energy equation of state') - - parser.add_argument('--H0', type=float, default=None, required=True, - help='Hubble constant') - - parser.add_argument('--sigma8', type=float, default=None, required=True, - help='Fluctuation amplitude on 8 Mpc/h scale') - - parser.add_argument('--zdrag', type=float, default=None, required=True, - help='Redshift at the drag epoch') - - parser.add_argument('--rdrag', type=float, default=None, required=True, - help='Sound horizon at the drag epoch') - - - args=parser.parse_args(cmdargs) - - pk=np.loadtxt(args.prefix_pk+'_matterpower.dat') - pkSB=np.loadtxt(args.prefix_pk+'SB_matterpower.dat') - col1=fits.Column(name='K',format='D',array=np.array(pk[:,0])) - col2=fits.Column(name='PK',format='D',array=np.array(pk[:,1])) - col3=fits.Column(name='PKSB',format='D',array=np.array(pkSB[:,1])) - cols=fits.ColDefs([col1,col2,col3]) - head=fits.Header() - head['ZREF']=args.zref - head['Om']=args.Om - head['Ok']=args.Ok - head['OL']=1.-args.Om-args.Ok - head['w']=args.w - head['H0']=args.H0 - head['sigma8']=args.sigma8 - head['zdrag']=args.zdrag - head['rdrag']=args.rdrag - tbhdu=fits.BinTableHDU.from_columns(cols,header=head) - tbhdu.writeto(args.out,clobber=True) +import numpy as np +from astropy.io import fits -if __name__ == '__main__': - cmdargs=sys.argv[1:] - main(cmdargs) \ No newline at end of file +def main(cmdargs): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "--prefix-pk", type=str, default=None, required=True, help="Prefix to pk file" + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--zref", type=float, default=None, required=True, help="Reference redshift" + ) + + parser.add_argument( + "--Om", type=float, default=None, required=True, help="Matter density parameter" + ) + + parser.add_argument( + "--Ok", + type=float, + default=None, + required=True, + help="Curvature density parameter", + ) + + parser.add_argument( + "--w", + type=float, + default=None, + required=True, + help="Dark energy equation of state", + ) + + parser.add_argument( + "--H0", type=float, default=None, required=True, help="Hubble constant" + ) + + parser.add_argument( + "--sigma8", + type=float, + default=None, + required=True, + help="Fluctuation amplitude on 8 Mpc/h scale", + ) + + parser.add_argument( + "--zdrag", + type=float, + default=None, + required=True, + help="Redshift at the drag epoch", + ) + + parser.add_argument( + "--rdrag", + type=float, + default=None, + required=True, + help="Sound horizon at the drag epoch", + ) + + args = parser.parse_args(cmdargs) + + pk = np.loadtxt(args.prefix_pk + "_matterpower.dat") + pkSB = np.loadtxt(args.prefix_pk + "SB_matterpower.dat") + col1 = fits.Column(name="K", format="D", array=np.array(pk[:, 0])) + col2 = fits.Column(name="PK", format="D", array=np.array(pk[:, 1])) + col3 = fits.Column(name="PKSB", format="D", array=np.array(pkSB[:, 1])) + cols = fits.ColDefs([col1, col2, col3]) + head = fits.Header() + head["ZREF"] = args.zref + head["Om"] = args.Om + head["Ok"] = args.Ok + head["OL"] = 1.0 - args.Om - args.Ok + head["w"] = args.w + head["H0"] = args.H0 + head["sigma8"] = args.sigma8 + head["zdrag"] = args.zdrag + head["rdrag"] = args.rdrag + tbhdu = fits.BinTableHDU.from_columns(cols, header=head) + tbhdu.writeto(args.out, clobber=True) + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_reduce_spall.py b/py/picca/bin/picca_reduce_spall.py old mode 100644 new mode 100755 index e4af2a2e1..6b863ce8c --- a/py/picca/bin/picca_reduce_spall.py +++ b/py/picca/bin/picca_reduce_spall.py @@ -1,36 +1,44 @@ -#!/usr/bin/env python -from astropy.table import Table +#!/usr/bin/env python3 import argparse +import sys + import numpy as np +from astropy.table import Table + + +def main(cmdargs=None): + parser = argparse.ArgumentParser() + parser.add_argument("--spall", type=str, required=True, help="Path to spAll file") + parser.add_argument( + "--qso-catalog", + type=str, + required=True, + help="Path to file containing THING_ID (e.g. DR16Q.fits)", + ) + parser.add_argument( + "--output", type=str, required=True, help="Path to output reduced spAll file" + ) + + args = parser.parse_args(cmdargs) + + print("Reading spAll file from") + print(args.spall) + spall = Table.read(args.spall) + print(f"{len(spall)} entries found in spAll file") + print("Reading QSO catalog from") + print(args.qso_catalog) + qso_catalog = Table.read(args.qso_catalog) + print(f"{len(qso_catalog)} entries found in QSO catalog") + + w = np.isin(spall["THING_ID"], qso_catalog["THING_ID"]) + spall_qso = spall[w] + # -- Columns required for picca_deltas.py for spec, spplate formats and usage of multiple observations + spall_qso.keep_columns( + ["THING_ID", "PLATE", "MJD", "FIBERID", "PLATEQUALITY", "ZWARNING"] + ) + spall_qso.write(args.output, overwrite=True) + -parser = argparse.ArgumentParser() -parser.add_argument('--spall', - type=str, - required=True, - help='Path to spAll file') -parser.add_argument('--qso-catalog', - type=str, - required=True, - help='Path to file containing THING_ID (e.g. DR16Q.fits)') -parser.add_argument('--output', - type=str, - required=True, - help='Path to output reduced spAll file') - -args = parser.parse_args() - -print('Reading spAll file from') -print(args.spall) -spall = Table.read(args.spall) -print(f'{len(spall)} entries found in spAll file') -print('Reading QSO catalog from') -print(args.qso_catalog) -qso_catalog = Table.read(args.qso_catalog) -print(f'{len(qso_catalog)} entries found in QSO catalog') - -w = np.isin(spall['THING_ID'], qso_catalog['THING_ID']) -spall_qso = spall[w] -#-- Columns required for picca_deltas.py for spec, spplate formats and usage of multiple observations -spall_qso.keep_columns( - ['THING_ID', 'PLATE', 'MJD', 'FIBERID', 'PLATEQUALITY', 'ZWARNING']) -spall_qso.write(args.output, overwrite=True) +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_wick.py b/py/picca/bin/picca_wick.py old mode 100644 new mode 100755 index 1d2113d4d..83fb6545c --- a/py/picca/bin/picca_wick.py +++ b/py/picca/bin/picca_wick.py @@ -1,17 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the wick covariance for the auto-correlation of forests The wick covariance is computed as explained in Delubac et al. 2015 """ -import sys import argparse import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value +import sys +from multiprocessing import Lock, Value, cpu_count + import fitsio import numpy as np from scipy.interpolate import interp1d -from picca import constants, cf, utils, io +from picca import cf, constants, io, utils from picca.utils import userprint @@ -40,242 +41,288 @@ def main(cmdargs): """Computes the wick covariance for the auto-correlation of forests""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the wick covariance for the auto-correlation of ' - 'forests')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--in-dir2', - type=str, - default=None, - required=False, - help='Directory to 2nd delta files') - - parser.add_argument('--rp-min', - type=float, - default=0., - required=False, - help='Min r-parallel [h^-1 Mpc]') - - parser.add_argument('--rp-max', - type=float, - default=200., - required=False, - help='Max r-parallel [h^-1 Mpc]') - - parser.add_argument('--rt-max', - type=float, - default=200., - required=False, - help='Max r-transverse [h^-1 Mpc]') - - parser.add_argument('--np', - type=int, - default=50, - required=False, - help='Number of r-parallel bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of r-transverse bins') - - parser.add_argument( - '--z-cut-min', + description=( + "Compute the wick covariance for the auto-correlation of " "forests" + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--in-dir2", + type=str, + default=None, + required=False, + help="Directory to 2nd delta files", + ) + + parser.add_argument( + "--rp-min", type=float, - default=0., + default=0.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help="Min r-parallel [h^-1 Mpc]", + ) parser.add_argument( - '--z-cut-max', + "--rp-max", type=float, - default=10., + default=200.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) + help="Max r-parallel [h^-1 Mpc]", + ) parser.add_argument( - '--z-min-sources', + "--rt-max", type=float, - default=0., + default=200.0, required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) + help="Max r-transverse [h^-1 Mpc]", + ) + + parser.add_argument( + "--np", type=int, default=50, required=False, help="Number of r-parallel bins" + ) parser.add_argument( - '--z-max-sources', + "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" + ) + + parser.add_argument( + "--z-cut-min", type=float, - default=10., + default=0.0, required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--lambda-abs', + "--z-cut-max", + type=float, + default=10.0, + required=False, + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) + + parser.add_argument( + "--z-min-sources", + type=float, + default=0.0, + required=False, + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) + + parser.add_argument( + "--z-max-sources", + type=float, + default=10.0, + required=False, + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) + + parser.add_argument( + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) parser.add_argument( - '--lambda-abs2', + "--lambda-abs2", type=str, default=None, required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the 2nd delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the 2nd delta" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol', + "--z-evol", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol2', + "--z-evol2", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the 2nd delta field') + help="Exponent of the redshift evolution of the 2nd delta field", + ) parser.add_argument( - '--fid-Om', + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--no-project', - action='store_true', - required=False, - help='Do not project out continuum fitting modes') + parser.add_argument( + "--no-project", + action="store_true", + required=False, + help="Do not project out continuum fitting modes", + ) - parser.add_argument('--max-diagram', - type=int, - default=3, - required=False, - help='Maximum diagram to compute') + parser.add_argument( + "--max-diagram", + type=int, + default=3, + required=False, + help="Maximum diagram to compute", + ) parser.add_argument( - '--cf1d', + "--cf1d", type=str, required=True, - help=('1D auto-correlation of pixels from the same forest file: ' - 'picca_cf1d.py')) + help=( + "1D auto-correlation of pixels from the same forest file: " "picca_cf1d.py" + ), + ) parser.add_argument( - '--cf1d2', + "--cf1d2", type=str, default=None, required=False, - help=('1D auto-correlation of pixels from the same forest file of the ' - '2nd delta field: picca_cf1d.py')) + help=( + "1D auto-correlation of pixels from the same forest file of the " + "2nd delta field: picca_cf1d.py" + ), + ) parser.add_argument( - '--cf', + "--cf", type=str, default=None, required=False, - help=('3D auto-correlation of pixels from different forests: ' - 'picca_cf.py')) + help=("3D auto-correlation of pixels from different forests: " "picca_cf.py"), + ) parser.add_argument( - '--cf2', + "--cf2", type=str, default=None, required=False, - help=('3D auto-correlation of pixels from different forests for 2nd ' - 'catalog: picca_cf.py')) + help=( + "3D auto-correlation of pixels from different forests for 2nd " + "catalog: picca_cf.py" + ), + ) parser.add_argument( - '--cf12', + "--cf12", type=str, default=None, required=False, - help=('3D auto-correlation of pixels from different forests for cross ' - '1st and 2nd catalog: picca_cf.py')) + help=( + "3D auto-correlation of pixels from different forests for cross " + "1st and 2nd catalog: picca_cf.py" + ), + ) parser.add_argument( - '--unfold-cf', - action='store_true', + "--unfold-cf", + action="store_true", required=False, - help=('rp can be positive or negative depending on the relative ' - 'position between absorber1 and absorber2')) + help=( + "rp can be positive or negative depending on the relative " + "position between absorber1 and absorber2" + ), + ) parser.add_argument( - '--rej', + "--rej", type=float, - default=1., + default=1.0, required=False, - help='Fraction of rejected pairs: -1=no rejection, 1=all rejection') - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') - - parser.add_argument('--rebin-factor', - type=int, - default=None, - required=False, - help='Rebin factor for deltas. If not None, deltas will ' - 'be rebinned by that factor') + help="Fraction of rejected pairs: -1=no rejection, 1=all rejection", + ) + + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) + + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, + required=False, + help="Maximum number of spectra to read", + ) + + parser.add_argument( + "--rebin-factor", + type=int, + default=None, + required=False, + help="Rebin factor for deltas. If not None, deltas will " + "be rebinned by that factor", + ) args = parser.parse_args(cmdargs) @@ -304,33 +351,45 @@ def main(cmdargs): blinding = io.read_blinding(args.in_dir) # load cosmology - if (args.fid_Or != 0.) or (args.fid_Ok != 0.) or (args.fid_wl != -1.): - userprint(("ERROR: Cosmology with other than Omega_m set are not yet " - "implemented")) + if (args.fid_Or != 0.0) or (args.fid_Ok != 0.0) or (args.fid_wl != -1.0): + userprint( + ("ERROR: Cosmology with other than Omega_m set are not yet " "implemented") + ) sys.exit() - cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl,) + cosmo = constants.Cosmo( + Om=args.fid_Om, + Or=args.fid_Or, + Ok=args.fid_Ok, + wl=args.fid_wl, + ) # read data 1 - data, num_data, z_min, z_max = io.read_deltas(args.in_dir, - cf.nside, - cf.lambda_abs, - cf.alpha, - cf.z_ref, - cosmo, - max_num_spec=args.nspec, - nproc=args.nproc, - rebin_factor=args.rebin_factor, - z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + data, num_data, z_min, z_max = io.read_deltas( + args.in_dir, + cf.nside, + cf.lambda_abs, + cf.alpha, + cf.z_ref, + cosmo, + max_num_spec=args.nspec, + nproc=args.nproc, + rebin_factor=args.rebin_factor, + z_min_qso=args.z_min_sources, + z_max_qso=args.z_max_sources, + ) for deltas in data.values(): for delta in deltas: - delta.fname = 'D1' + delta.fname = "D1" for item in [ - 'cont', 'delta', 'order', 'ivar', 'exposures_diff', - 'mean_snr', 'mean_reso', 'mean_z', 'delta_log_lambda' + "cont", + "delta", + "order", + "ivar", + "exposures_diff", + "mean_snr", + "mean_reso", + "mean_z", + "delta_log_lambda", ]: setattr(delta, item, None) del z_max @@ -347,25 +406,26 @@ def main(cmdargs): continue hdul = fitsio.FITS(filename) header = hdul[1].read_header() - log_lambda_min = header['LLMIN'] - delta_log_lambda = header['DLL'] - num_pairs_variance_1d = hdul[1]['nv1d'][:] - variance_1d = hdul[1]['v1d'][:] - log_lambda = (log_lambda_min + - delta_log_lambda * np.arange(len(variance_1d))) + log_lambda_min = header["LLMIN"] + delta_log_lambda = header["DLL"] + num_pairs_variance_1d = hdul[1]["nv1d"][:] + variance_1d = hdul[1]["v1d"][:] + log_lambda = log_lambda_min + delta_log_lambda * np.arange(len(variance_1d)) cf.get_variance_1d[fname] = interp1d( log_lambda[num_pairs_variance_1d > 0], variance_1d[num_pairs_variance_1d > 0], - kind='nearest', - fill_value='extrapolate') + kind="nearest", + fill_value="extrapolate", + ) - num_pairs1d = hdul[1]['nb1d'][:] - xi_1d = hdul[1]['c1d'][:] + num_pairs1d = hdul[1]["nb1d"][:] + xi_1d = hdul[1]["c1d"][:] cf.xi_1d[fname] = interp1d( (log_lambda - log_lambda_min)[num_pairs1d > 0], xi_1d[num_pairs1d > 0], - kind='nearest', - fill_value='extrapolate') + kind="nearest", + fill_value="extrapolate", + ) hdul.close() # Load correlation functions @@ -373,23 +433,23 @@ def main(cmdargs): "D1_D1": args.cf, "D2_D2": args.cf2, "D1_D2": args.cf12, - "D2_D1": args.cf12 + "D2_D1": args.cf12, } for fname, filename in dic_xi.items(): if filename is None: continue hdul = fitsio.FITS(filename) header = hdul[1].read_header() - assert cf.num_bins_r_par == header['NP'] - assert cf.num_bins_r_trans == header['NT'] - assert cf.r_par_min == header['RPMIN'] - assert cf.r_par_max == header['RPMAX'] - assert cf.r_trans_max == header['RTMAX'] - xi = hdul[2]['DA'][:] - weights = hdul[2]['WE'][:] + assert cf.num_bins_r_par == header["NP"] + assert cf.num_bins_r_trans == header["NT"] + assert cf.r_par_min == header["RPMIN"] + assert cf.r_par_max == header["RPMAX"] + assert cf.r_trans_max == header["RTMAX"] + xi = hdul[2]["DA"][:] + weights = hdul[2]["WE"][:] xi = (xi * weights).sum(axis=0) weights = weights.sum(axis=0) - w = weights > 0. + w = weights > 0.0 xi[w] /= weights[w] cf.xi_wick[fname] = xi.copy() hdul.close() @@ -418,13 +478,21 @@ def main(cmdargs): nproc=args.nproc, rebin_factor=args.rebin_factor, z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + z_max_qso=args.z_max_sources, + ) for deltas in data.values(): for delta in deltas: - delta.fname = 'D2' + delta.fname = "D2" for item in [ - 'cont', 'delta', 'order', 'ivar', 'exposures_diff', - 'mean_snr', 'mean_reso', 'mean_z', 'delta_log_lambda' + "cont", + "delta", + "order", + "ivar", + "exposures_diff", + "mean_snr", + "mean_reso", + "mean_z", + "delta_log_lambda", ]: setattr(delta, item, None) del z_max2 @@ -434,7 +502,7 @@ def main(cmdargs): userprint("") userprint("done, npix = {}".format(len(data2))) - cf.counter = Value('i', 0) + cf.counter = Value("i", 0) cf.lock = Lock() cpu_data = {} @@ -445,10 +513,10 @@ def main(cmdargs): cpu_data[num_processor].append(healpix) # compute the covariance matrix - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=min(args.nproc, len(cpu_data.values()))) userprint(" \nStarting\n") - if args.nproc>1: + if args.nproc > 1: wick_data = pool.map(calc_wick_terms, sorted(cpu_data.values())) else: wick_data = [calc_wick_terms(arg) for arg in sorted(cpu_data.values())] @@ -468,108 +536,109 @@ def main(cmdargs): t5 = np.array([item[8] for item in wick_data]).sum(axis=0) t6 = np.array([item[9] for item in wick_data]).sum(axis=0) weights = weights_wick * weights_wick[:, None] - w = weights > 0. + w = weights > 0.0 t1[w] /= weights[w] t2[w] /= weights[w] t3[w] /= weights[w] t4[w] /= weights[w] t5[w] /= weights[w] t6[w] /= weights[w] - t1 *= 1. * num_pairs_used / num_pairs - t2 *= 1. * num_pairs_used / num_pairs - t3 *= 1. * num_pairs_used / num_pairs - t4 *= 1. * num_pairs_used / num_pairs - t5 *= 1. * num_pairs_used / num_pairs - t6 *= 1. * num_pairs_used / num_pairs + t1 *= 1.0 * num_pairs_used / num_pairs + t2 *= 1.0 * num_pairs_used / num_pairs + t3 *= 1.0 * num_pairs_used / num_pairs + t4 *= 1.0 * num_pairs_used / num_pairs + t5 *= 1.0 * num_pairs_used / num_pairs + t6 *= 1.0 * num_pairs_used / num_pairs t_tot = t1 + t2 + t3 + t4 + t5 + t6 # save results - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = [ { - 'name': 'RPMIN', - 'value': cf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' + "name": "RPMIN", + "value": cf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": cf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", }, { - 'name': 'RPMAX', - 'value': cf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' + "name": "RTMAX", + "value": cf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", }, { - 'name': 'RTMAX', - 'value': cf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' + "name": "NP", + "value": cf.num_bins_r_par, + "comment": "Number of bins in r-parallel", }, { - 'name': 'NP', - 'value': cf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' + "name": "NT", + "value": cf.num_bins_r_trans, + "comment": "Number of bins in r-transverse", }, { - 'name': 'NT', - 'value': cf.num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' + "name": "ZCUTMIN", + "value": cf.z_cut_min, + "comment": "Minimum redshift of pairs", }, { - 'name': 'ZCUTMIN', - 'value': cf.z_cut_min, - 'comment': 'Minimum redshift of pairs' + "name": "ZCUTMAX", + "value": cf.z_cut_max, + "comment": "Maximum redshift of pairs", }, + {"name": "REJ", "value": cf.reject, "comment": "Rejection factor"}, + {"name": "NPALL", "value": num_pairs, "comment": "Number of pairs"}, + {"name": "NPUSED", "value": num_pairs_used, "comment": "Number of used pairs"}, { - 'name': 'ZCUTMAX', - 'value': cf.z_cut_max, - 'comment': 'Maximum redshift of pairs' + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'REJ', - 'value': cf.reject, - 'comment': 'Rejection factor' + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'NPALL', - 'value': num_pairs, - 'comment': 'Number of pairs' + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'NPUSED', - 'value': num_pairs_used, - 'comment': 'Number of used pairs' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - } - ] + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + ] comment = [ - 'Sum of weight', 'Covariance', 'Nomber of pairs', 'T1', 'T2', 'T3', - 'T4', 'T5', 'T6' + "Sum of weight", + "Covariance", + "Nomber of pairs", + "T1", + "T2", + "T3", + "T4", + "T5", + "T6", ] results.write( [t_tot, weights_wick, num_pairs_wick, t1, t2, t3, t4, t5, t6], - names=['CO', 'WALL', 'NB', 'T1', 'T2', 'T3', 'T4', 'T5', 'T6'], + names=["CO", "WALL", "NB", "T1", "T2", "T3", "T4", "T5", "T6"], comment=comment, header=header, - extname='COV') + extname="COV", + ) results.close() -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_write_full_covariance.py b/py/picca/bin/picca_write_full_covariance.py old mode 100644 new mode 100755 index defbcd396..cd4296f60 --- a/py/picca/bin/picca_write_full_covariance.py +++ b/py/picca/bin/picca_write_full_covariance.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Compute and write out the full (unsmoothed) covariance matrix given some correlation functions. @@ -7,6 +7,7 @@ """ import argparse +import sys import time from functools import reduce @@ -43,7 +44,7 @@ def read_corr(files): def main(cmdargs=None): - + parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Writes the full covariance matrix for the given correlation functions.", @@ -98,3 +99,8 @@ def main(cmdargs=None): print() print(f"Time spent computing covariance: {(t3 - t2)/60:.3f} minutes") print(f"Total time: {(t4-t1)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_write_smooth_covariance.py b/py/picca/bin/picca_write_smooth_covariance.py old mode 100644 new mode 100755 index b59218913..fb12e7455 --- a/py/picca/bin/picca_write_smooth_covariance.py +++ b/py/picca/bin/picca_write_smooth_covariance.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Smooth the covariance matrix calculated from the 'write_full_covariance' script. @@ -7,6 +7,7 @@ """ import argparse +import sys import time import fitsio @@ -72,7 +73,7 @@ def smooth_corrmat_asym( def main(cmdargs=None): - + parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Smoothes the covariance matrix.", @@ -311,3 +312,8 @@ def main(cmdargs=None): print(f"Time spent smoothing covariance: {(t3 - t2)/60:.3f} minutes") print(f"Total time: {(t3 - t1)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_xcf.py b/py/picca/bin/picca_xcf.py old mode 100644 new mode 100755 index 11f698a0d..73f150d02 --- a/py/picca/bin/picca_xcf.py +++ b/py/picca/bin/picca_xcf.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the auto and cross-correlation between a catalog of objects and a delta field. @@ -7,8 +7,9 @@ """ import argparse import multiprocessing +import sys import time -from multiprocessing import Lock, Pool, Value, cpu_count +from multiprocessing import Lock, Value, cpu_count import fitsio import numpy as np @@ -109,16 +110,22 @@ def main(cmdargs=None): "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" ) - parser.add_argument('--rmu-binning', action="store_true", - help=('Estimate in r,mu binning. np becomes mu bins.' - ' nt becomes r bins. rp min max is always -1, 1') - ) + parser.add_argument( + "--rmu-binning", + action="store_true", + help=( + "Estimate in r,mu binning. np becomes mu bins." + " nt becomes r bins. rp min max is always -1, 1" + ), + ) - parser.add_argument('--z-min-obj', - type=float, - default=0, - required=False, - help='Min redshift for object field') + parser.add_argument( + "--z-min-obj", + type=float, + default=0, + required=False, + help="Min redshift for object field", + ) parser.add_argument( "--z-max-obj", @@ -513,72 +520,85 @@ def main(cmdargs=None): z[w] /= sum_weights[w] num_pairs = num_pairs_list.sum(axis=0) - results = fitsio.FITS(args.out, 'rw', clobber=True) - header = [{ - 'name': 'RPMIN', - 'value': xcf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RPMAX', - 'value': xcf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' - }, { - 'name': 'RTMAX', - 'value': xcf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' - }, { - 'name': 'NP', - 'value': xcf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' - }, { - 'name': 'NT', - 'value': xcf.num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' - }, { - 'name': 'ZCUTMIN', - 'value': xcf.z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, { - 'name': 'ZCUTMAX', - 'value': xcf.z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, { - 'name': 'NSIDE', - 'value': xcf.nside, - 'comment': 'Healpix nside' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - }, { - 'name': "RMU_BIN", - 'value': xcf.rmu_binning, - 'comment': 'True if binned in r, mu' - } + results = fitsio.FITS(args.out, "rw", clobber=True) + header = [ + { + "name": "RPMIN", + "value": xcf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": xcf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", + }, + { + "name": "RTMAX", + "value": xcf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", + }, + { + "name": "NP", + "value": xcf.num_bins_r_par, + "comment": "Number of bins in r-parallel", + }, + { + "name": "NT", + "value": xcf.num_bins_r_trans, + "comment": "Number of bins in r-transverse", + }, + { + "name": "ZCUTMIN", + "value": xcf.z_cut_min, + "comment": "Minimum redshift of pairs", + }, + { + "name": "ZCUTMAX", + "value": xcf.z_cut_max, + "comment": "Maximum redshift of pairs", + }, + {"name": "NSIDE", "value": xcf.nside, "comment": "Healpix nside"}, + { + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + { + "name": "RMU_BIN", + "value": xcf.rmu_binning, + "comment": "True if binned in r, mu", + }, ] results.write( [r_par, r_trans, z, num_pairs], - names=['RP', 'RT', 'Z', 'NB'], - comment=['R-parallel' if not xcf.rmu_binning else 'Mu', - 'R-transverse' if not xcf.rmu_binning else 'Radial separation', - 'Redshift', 'Number of pairs'], - units=['h^-1 Mpc' if not xcf.rmu_binning else '', 'h^-1 Mpc', '', ''], + names=["RP", "RT", "Z", "NB"], + comment=[ + "R-parallel" if not xcf.rmu_binning else "Mu", + "R-transverse" if not xcf.rmu_binning else "Radial separation", + "Redshift", + "Number of pairs", + ], + units=["h^-1 Mpc" if not xcf.rmu_binning else "", "h^-1 Mpc", "", ""], header=header, extname="ATTRI", ) @@ -599,3 +619,8 @@ def main(cmdargs=None): t3 = time.time() userprint(f"picca_xcf.py - Time total: {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_xcf1d.py b/py/picca/bin/picca_xcf1d.py old mode 100644 new mode 100755 index 3fc91e6d7..8b3ef4591 --- a/py/picca/bin/picca_xcf1d.py +++ b/py/picca/bin/picca_xcf1d.py @@ -1,15 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the 1D auto or cross-correlation between a catalog of objects and a delta field as a function of wavelength ratio """ import argparse -import sys import multiprocessing +import sys from multiprocessing import Pool, cpu_count -import numpy as np + import fitsio +import numpy as np -from picca import constants, xcf, io, prep_del +from picca import constants, io, prep_del, xcf from picca.data import Forest from picca.utils import userprint @@ -32,163 +33,196 @@ def main(cmdargs): field as a function of wavelength ratio""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the 1D cross-correlation between a catalog of ' - 'objects and a delta field as a function of wavelength ' - 'ratio')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--drq', - type=str, - default=None, - required=True, - help='Catalog of objects in DRQ format') + description=( + "Compute the 1D cross-correlation between a catalog of " + "objects and a delta field as a function of wavelength " + "ratio" + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--drq", + type=str, + default=None, + required=True, + help="Catalog of objects in DRQ format", + ) + + parser.add_argument( + "--mode", + type=str, + default="sdss", + choices=["sdss", "desi"], + required=False, + help="type of catalog supplied, default sdss", + ) + + parser.add_argument( + "--wr-min", + type=float, + default=0.9, + required=False, + help="Min of wavelength ratio", + ) + + parser.add_argument( + "--wr-max", + type=float, + default=1.1, + required=False, + help="Max of wavelength ratio", + ) + + parser.add_argument( + "--np", + type=int, + default=100, + required=False, + help="Number of wavelength ratio bins", + ) + + parser.add_argument( + "--z-min-obj", + type=float, + default=0, + required=False, + help="Min redshift for object field", + ) parser.add_argument( - '--mode', - type=str, - default='sdss', - choices=['sdss','desi'], - required=False, - help='type of catalog supplied, default sdss') - - parser.add_argument('--wr-min', - type=float, - default=0.9, - required=False, - help='Min of wavelength ratio') - - parser.add_argument('--wr-max', - type=float, - default=1.1, - required=False, - help='Max of wavelength ratio') - - parser.add_argument('--np', - type=int, - default=100, - required=False, - help='Number of wavelength ratio bins') - - parser.add_argument('--z-min-obj', - type=float, - default=0, - required=False, - help='Min redshift for object field') - - parser.add_argument('--z-max-obj', - type=float, - default=10, - required=False, - help='Max redshift for object field') + "--z-max-obj", + type=float, + default=10, + required=False, + help="Max redshift for object field", + ) parser.add_argument( - '--z-cut-min', + "--z-cut-min", type=float, - default=0., + default=0.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--z-cut-max', + "--z-cut-max", type=float, - default=10., + default=10.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) parser.add_argument( - '--z-min-sources', + "--z-min-sources", type=float, - default=0., + default=0.0, required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--z-max-sources', + "--z-max-sources", type=float, - default=10., + default=10.0, required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--lambda-abs', + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) parser.add_argument( - '--lambda-abs-obj', + "--lambda-abs-obj", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants the object is ' - 'considered as')) + help=( + "Name of the absorption in picca.constants the object is " "considered as" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol-del', + "--z-evol-del", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol-obj', + "--z-evol-obj", type=float, default=1.44, required=False, - help='Exponent of the redshift evolution of the object field') - - parser.add_argument('--no-project', - action='store_true', - required=False, - help='Do not project out continuum fitting modes') - - parser.add_argument('--no-remove-mean-lambda-obs', - action='store_true', - required=False, - help='Do not remove mean delta versus lambda_obs') - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') + help="Exponent of the redshift evolution of the object field", + ) + + parser.add_argument( + "--no-project", + action="store_true", + required=False, + help="Do not project out continuum fitting modes", + ) + + parser.add_argument( + "--no-remove-mean-lambda-obs", + action="store_true", + required=False, + help="Do not remove mean delta versus lambda_obs", + ) + + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) + + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, + required=False, + help="Maximum number of spectra to read", + ) args = parser.parse_args(cmdargs) if args.nproc is None: @@ -197,7 +231,7 @@ def main(cmdargs): # setup variables in module xcf xcf.r_par_min = args.wr_min xcf.r_par_max = args.wr_max - xcf.r_trans_max = 1.e-6 + xcf.r_trans_max = 1.0e-6 xcf.z_cut_min = args.z_cut_min xcf.z_cut_max = args.z_cut_max xcf.num_bins_r_par = args.np @@ -208,16 +242,18 @@ def main(cmdargs): lambda_abs = constants.ABSORBER_IGM[args.lambda_abs] ### Read deltas - data, num_data, z_min, z_max = io.read_deltas(args.in_dir, - args.nside, - lambda_abs, - args.z_evol_del, - args.z_ref, - cosmo=None, - max_num_spec=args.nspec, - no_project=args.no_project, - z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + data, num_data, z_min, z_max = io.read_deltas( + args.in_dir, + args.nside, + lambda_abs, + args.z_evol_del, + args.z_ref, + cosmo=None, + max_num_spec=args.nspec, + no_project=args.no_project, + z_min_qso=args.z_min_sources, + z_max_qso=args.z_max_sources, + ) xcf.data = data xcf.num_data = num_data sys.stderr.write("\n") @@ -228,47 +264,58 @@ def main(cmdargs): Forest.delta_log_lambda = None for healpix in xcf.data: for delta in xcf.data[healpix]: - delta_log_lambda = np.asarray([ - delta.log_lambda[index] - delta.log_lambda[index - 1] - for index in range(1, delta.log_lambda.size) - ]).min() + delta_log_lambda = np.asarray( + [ + delta.log_lambda[index] - delta.log_lambda[index - 1] + for index in range(1, delta.log_lambda.size) + ] + ).min() if Forest.delta_log_lambda is None: Forest.delta_log_lambda = delta_log_lambda else: - Forest.delta_log_lambda = min(delta_log_lambda, - Forest.delta_log_lambda) - Forest.log_lambda_min = (np.log10( - (z_min + 1.) * lambda_abs) - Forest.delta_log_lambda / 2.) - Forest.log_lambda_max = (np.log10( - (z_max + 1.) * lambda_abs) + Forest.delta_log_lambda / 2.) + Forest.delta_log_lambda = min( + delta_log_lambda, Forest.delta_log_lambda + ) + Forest.log_lambda_min = ( + np.log10((z_min + 1.0) * lambda_abs) - Forest.delta_log_lambda / 2.0 + ) + Forest.log_lambda_max = ( + np.log10((z_max + 1.0) * lambda_abs) + Forest.delta_log_lambda / 2.0 + ) log_lambda, mean_delta, stack_weight = prep_del.stack( - xcf.data, stack_from_deltas=True) + xcf.data, stack_from_deltas=True + ) del log_lambda, stack_weight for healpix in xcf.data: for delta in xcf.data[healpix]: - bins = ((delta.log_lambda - Forest.log_lambda_min) / - Forest.delta_log_lambda + 0.5).astype(int) + bins = ( + (delta.log_lambda - Forest.log_lambda_min) / Forest.delta_log_lambda + + 0.5 + ).astype(int) delta.delta -= mean_delta[bins] ### Read objects - objs, z_min2 = io.read_objects(args.drq, - args.nside, - args.z_min_obj, - args.z_max_obj, - args.z_evol_obj, - args.z_ref, - cosmo=None, - mode=args.mode) + objs, z_min2 = io.read_objects( + args.drq, + args.nside, + args.z_min_obj, + args.z_max_obj, + args.z_evol_obj, + args.z_ref, + cosmo=None, + mode=args.mode, + ) del z_min2 xcf.objs = objs for healpix in xcf.objs: for obj in xcf.objs[healpix]: obj.log_lambda = np.log10( - (1. + obj.z_qso) * constants.ABSORBER_IGM[args.lambda_abs_obj]) + (1.0 + obj.z_qso) * constants.ABSORBER_IGM[args.lambda_abs_obj] + ) sys.stderr.write("\n") # Compute the correlation function, use pool to parallelize - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=args.nproc) healpixs = [[healpix] for healpix in sorted(data) if healpix in xcf.objs] correlation_function_data = pool.map(corr_func, healpixs) @@ -282,62 +329,66 @@ def main(cmdargs): z_list = correlation_function_data[:, 3, :] num_pairs_list = correlation_function_data[:, 4, :].astype(np.int64) healpix_list = np.array( - [healpix for healpix in sorted(data) if healpix in xcf.objs]) + [healpix for healpix in sorted(data) if healpix in xcf.objs] + ) - w = (weights_list.sum(axis=0) > 0.) + w = weights_list.sum(axis=0) > 0.0 r_par = (r_par_list * weights_list).sum(axis=0) r_par[w] /= weights_list.sum(axis=0)[w] z = (z_list * weights_list).sum(axis=0) z[w] /= weights_list.sum(axis=0)[w] num_pairs = num_pairs_list.sum(axis=0) - results = fitsio.FITS(args.out, 'rw', clobber=True) - header = [{ - 'name': 'RPMIN', - 'value': xcf.r_par_min, - 'comment': 'Minimum wavelength ratio' - }, { - 'name': 'RPMAX', - 'value': xcf.r_par_max, - 'comment': 'Maximum wavelength ratio' - }, { - 'name': 'NP', - 'value': xcf.num_bins_r_par, - 'comment': 'Number of bins in wavelength ratio' - }, { - 'name': 'ZCUTMIN', - 'value': xcf.z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, { - 'name': 'ZCUTMAX', - 'value': xcf.z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, { - 'name': 'NSIDE', - 'value': xcf.nside, - 'comment': 'Healpix nside' - }] - results.write([r_par, z, num_pairs], - names=['RP', 'Z', 'NB'], - units=['', '', ''], - comment=['Wavelength ratio', 'Redshift', 'Number of pairs'], - header=header, - extname='ATTRI') - - header2 = [{ - 'name': 'HLPXSCHM', - 'value': 'RING', - 'comment': 'Healpix scheme' - }] - results.write([healpix_list, weights_list, xi_list], - names=['HEALPID', 'WE', 'DA'], - comment=['Healpix index', 'Sum of weight', 'Correlation'], - header=header2, - extname='COR') + results = fitsio.FITS(args.out, "rw", clobber=True) + header = [ + { + "name": "RPMIN", + "value": xcf.r_par_min, + "comment": "Minimum wavelength ratio", + }, + { + "name": "RPMAX", + "value": xcf.r_par_max, + "comment": "Maximum wavelength ratio", + }, + { + "name": "NP", + "value": xcf.num_bins_r_par, + "comment": "Number of bins in wavelength ratio", + }, + { + "name": "ZCUTMIN", + "value": xcf.z_cut_min, + "comment": "Minimum redshift of pairs", + }, + { + "name": "ZCUTMAX", + "value": xcf.z_cut_max, + "comment": "Maximum redshift of pairs", + }, + {"name": "NSIDE", "value": xcf.nside, "comment": "Healpix nside"}, + ] + results.write( + [r_par, z, num_pairs], + names=["RP", "Z", "NB"], + units=["", "", ""], + comment=["Wavelength ratio", "Redshift", "Number of pairs"], + header=header, + extname="ATTRI", + ) + + header2 = [{"name": "HLPXSCHM", "value": "RING", "comment": "Healpix scheme"}] + results.write( + [healpix_list, weights_list, xi_list], + names=["HEALPID", "WE", "DA"], + comment=["Healpix index", "Sum of weight", "Correlation"], + header=header2, + extname="COR", + ) results.close() -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_xcf_angl.py b/py/picca/bin/picca_xcf_angl.py old mode 100644 new mode 100755 index ebb5ccf02..20e63bf78 --- a/py/picca/bin/picca_xcf_angl.py +++ b/py/picca/bin/picca_xcf_angl.py @@ -1,15 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the cross-correlation between a catalog of objects and a delta field as a function of angle and wavelength ratio """ -import sys import argparse import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value -import numpy as np +import sys +from multiprocessing import Lock, Pool, Value, cpu_count + import fitsio +import numpy as np -from picca import constants, xcf, io, prep_del +from picca import constants, io, prep_del, xcf from picca.data import Forest from picca.utils import userprint @@ -38,202 +39,236 @@ def main(cmdargs): field as a function of angle and wavelength ratio""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the cross-correlation between a catalog of ' - 'objects and a delta field as a function of angle and ' - 'wavelength ratio')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--drq', - type=str, - default=None, - required=True, - help='Catalog of objects in DRQ format') + description=( + "Compute the cross-correlation between a catalog of " + "objects and a delta field as a function of angle and " + "wavelength ratio" + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--drq", + type=str, + default=None, + required=True, + help="Catalog of objects in DRQ format", + ) + + parser.add_argument( + "--mode", + type=str, + default="sdss", + choices=["sdss", "desi"], + required=False, + help="type of catalog supplied, default sdss", + ) + + parser.add_argument( + "--wr-min", + type=float, + default=0.9, + required=False, + help="Min of wavelength ratio", + ) + + parser.add_argument( + "--wr-max", + type=float, + default=1.1, + required=False, + help="Max of wavelength ratio", + ) + + parser.add_argument( + "--ang-max", type=float, default=0.02, required=False, help="Max angle (rad)" + ) + + parser.add_argument( + "--np", + type=int, + default=100, + required=False, + help="Number of wavelength ratio bins", + ) + + parser.add_argument( + "--nt", type=int, default=50, required=False, help="Number of angular bins" + ) parser.add_argument( - '--mode', - type=str, - default='sdss', - choices=['sdss','desi'], - required=False, - help='type of catalog supplied, default sdss') - - parser.add_argument('--wr-min', - type=float, - default=0.9, - required=False, - help='Min of wavelength ratio') - - parser.add_argument('--wr-max', - type=float, - default=1.1, - required=False, - help='Max of wavelength ratio') - - parser.add_argument('--ang-max', - type=float, - default=0.02, - required=False, - help='Max angle (rad)') - - parser.add_argument('--np', - type=int, - default=100, - required=False, - help='Number of wavelength ratio bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of angular bins') - - parser.add_argument('--z-min-obj', - type=float, - default=0, - required=False, - help='Min redshift for object field') - - parser.add_argument('--z-max-obj', - type=float, - default=10, - required=False, - help='Max redshift for object field') + "--z-min-obj", + type=float, + default=0, + required=False, + help="Min redshift for object field", + ) parser.add_argument( - '--z-cut-min', + "--z-max-obj", type=float, - default=0., + default=10, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help="Max redshift for object field", + ) parser.add_argument( - '--z-cut-max', + "--z-cut-min", type=float, - default=10., + default=0.0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--z-min-sources', + "--z-cut-max", type=float, - default=0., + default=10.0, required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) parser.add_argument( - '--z-max-sources', + "--z-min-sources", type=float, - default=10., + default=0.0, required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) parser.add_argument( - '--lambda-abs', + "--z-max-sources", + type=float, + default=10.0, + required=False, + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) + + parser.add_argument( + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) parser.add_argument( - '--lambda-abs-obj', + "--lambda-abs-obj", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants the object is ' - 'considered as')) + help=( + "Name of the absorption in picca.constants the object is " "considered as" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol-del', + "--z-evol-del", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol-obj', + "--z-evol-obj", type=float, default=1.44, required=False, - help='Exponent of the redshift evolution of the object field') + help="Exponent of the redshift evolution of the object field", + ) parser.add_argument( - '--fid-Om', + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') - - parser.add_argument('--no-project', - action='store_true', - required=False, - help='Do not project out continuum fitting modes') - - parser.add_argument('--no-remove-mean-lambda-obs', - action='store_true', - required=False, - help='Do not remove mean delta versus lambda_obs') - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) + + parser.add_argument( + "--no-project", + action="store_true", + required=False, + help="Do not project out continuum fitting modes", + ) + + parser.add_argument( + "--no-remove-mean-lambda-obs", + action="store_true", + required=False, + help="Do not remove mean delta versus lambda_obs", + ) + + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) + + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, + required=False, + help="Maximum number of spectra to read", + ) args = parser.parse_args(cmdargs) if args.nproc is None: @@ -256,10 +291,12 @@ def main(cmdargs): blinding = io.read_blinding(args.in_dir) # load fiducial cosmology - cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl,) + cosmo = constants.Cosmo( + Om=args.fid_Om, + Or=args.fid_Or, + Ok=args.fid_Ok, + wl=args.fid_wl, + ) ### Read deltas data, num_data, z_min, z_max = io.read_deltas( @@ -272,7 +309,8 @@ def main(cmdargs): max_num_spec=args.nspec, no_project=args.no_project, z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + z_max_qso=args.z_max_sources, + ) xcf.data = data xcf.num_data = num_data userprint("") @@ -283,48 +321,63 @@ def main(cmdargs): Forest.delta_log_lambda = None for healpix in xcf.data: for delta in xcf.data[healpix]: - delta_log_lambda = np.asarray([ - delta.log_lambda[index] - delta.log_lambda[index - 1] - for index in range(1, delta.log_lambda.size) - ]).min() + delta_log_lambda = np.asarray( + [ + delta.log_lambda[index] - delta.log_lambda[index - 1] + for index in range(1, delta.log_lambda.size) + ] + ).min() if Forest.delta_log_lambda is None: Forest.delta_log_lambda = delta_log_lambda else: - Forest.delta_log_lambda = min(delta_log_lambda, - Forest.delta_log_lambda) - Forest.log_lambda_min = (np.log10( - (z_min + 1.) * xcf.lambda_abs) - Forest.delta_log_lambda / 2.) - Forest.log_lambda_max = (np.log10( - (z_max + 1.) * xcf.lambda_abs) + Forest.delta_log_lambda / 2.) + Forest.delta_log_lambda = min( + delta_log_lambda, Forest.delta_log_lambda + ) + Forest.log_lambda_min = ( + np.log10((z_min + 1.0) * xcf.lambda_abs) - Forest.delta_log_lambda / 2.0 + ) + Forest.log_lambda_max = ( + np.log10((z_max + 1.0) * xcf.lambda_abs) + Forest.delta_log_lambda / 2.0 + ) log_lambda, mean_delta, stack_weight = prep_del.stack( - xcf.data, stack_from_deltas=True) + xcf.data, stack_from_deltas=True + ) del log_lambda, stack_weight for healpix in xcf.data: for delta in xcf.data[healpix]: - bins = ((delta.log_lambda - Forest.log_lambda_min) / - Forest.delta_log_lambda + 0.5).astype(int) + bins = ( + (delta.log_lambda - Forest.log_lambda_min) / Forest.delta_log_lambda + + 0.5 + ).astype(int) delta.delta -= mean_delta[bins] ### Read objects - objs, z_min2 = io.read_objects(args.drq, args.nside, args.z_min_obj, - args.z_max_obj, args.z_evol_obj, args.z_ref, - cosmo, mode=args.mode) + objs, z_min2 = io.read_objects( + args.drq, + args.nside, + args.z_min_obj, + args.z_max_obj, + args.z_evol_obj, + args.z_ref, + cosmo, + mode=args.mode, + ) del z_min2 for index, healpix in enumerate(sorted(objs)): for obj in objs[healpix]: obj.log_lambda = np.log10( - (1. + obj.z_qso) * constants.ABSORBER_IGM[args.lambda_abs_obj]) + (1.0 + obj.z_qso) * constants.ABSORBER_IGM[args.lambda_abs_obj] + ) userprint("") xcf.objs = objs # compute correlation function, use pool to parallelize - xcf.counter = Value('i', 0) + xcf.counter = Value("i", 0) xcf.lock = Lock() cpu_data = {healpix: [healpix] for healpix in data} - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=args.nproc) - correlation_function_data = pool.map(corr_func, - sorted(list(cpu_data.values()))) + correlation_function_data = pool.map(corr_func, sorted(list(cpu_data.values()))) pool.close() # group data from parallelisation @@ -337,7 +390,7 @@ def main(cmdargs): num_pairs_list = correlation_function_data[:, 5, :].astype(np.int64) healpix_list = np.array(sorted(list(cpu_data.keys()))) - w = (weights_list.sum(axis=0) > 0.) + w = weights_list.sum(axis=0) > 0.0 r_par = (r_par_list * weights_list).sum(axis=0) r_par[w] /= weights_list.sum(axis=0)[w] r_trans = (r_trans_list * weights_list).sum(axis=0) @@ -347,65 +400,65 @@ def main(cmdargs): num_pairs = num_pairs_list.sum(axis=0) # save results - results = fitsio.FITS(args.out, 'rw', clobber=True) - header = [{ - 'name': 'RPMIN', - 'value': xcf.r_par_min, - 'comment': 'Minimum wavelength ratio' - }, { - 'name': 'RPMAX', - 'value': xcf.r_par_max, - 'comment': 'Maximum wavelength ratio' - }, { - 'name': 'RTMAX', - 'value': xcf.r_trans_max, - 'comment': 'Maximum angle [rad]' - }, { - 'name': 'NP', - 'value': xcf.num_bins_r_par, - 'comment': 'Number of bins in wavelength ratio' - }, { - 'name': 'NT', - 'value': xcf.num_bins_r_trans, - 'comment': 'Number of bins in angle' - }, { - 'name': 'ZCUTMIN', - 'value': xcf.z_cut_min, - 'comment': 'Minimum redshift of pairs' - }, { - 'name': 'ZCUTMAX', - 'value': xcf.z_cut_max, - 'comment': 'Maximum redshift of pairs' - }, { - 'name': 'NSIDE', - 'value': xcf.nside, - 'comment': 'Healpix nside' - }] + results = fitsio.FITS(args.out, "rw", clobber=True) + header = [ + { + "name": "RPMIN", + "value": xcf.r_par_min, + "comment": "Minimum wavelength ratio", + }, + { + "name": "RPMAX", + "value": xcf.r_par_max, + "comment": "Maximum wavelength ratio", + }, + {"name": "RTMAX", "value": xcf.r_trans_max, "comment": "Maximum angle [rad]"}, + { + "name": "NP", + "value": xcf.num_bins_r_par, + "comment": "Number of bins in wavelength ratio", + }, + { + "name": "NT", + "value": xcf.num_bins_r_trans, + "comment": "Number of bins in angle", + }, + { + "name": "ZCUTMIN", + "value": xcf.z_cut_min, + "comment": "Minimum redshift of pairs", + }, + { + "name": "ZCUTMAX", + "value": xcf.z_cut_max, + "comment": "Maximum redshift of pairs", + }, + {"name": "NSIDE", "value": xcf.nside, "comment": "Healpix nside"}, + ] results.write( [r_par, r_trans, z, num_pairs], - names=['RP', 'RT', 'Z', 'NB'], - units=['', 'rad', '', ''], - comment=['Wavelength ratio', 'Angle', 'Redshift', 'Number of pairs'], + names=["RP", "RT", "Z", "NB"], + units=["", "rad", "", ""], + comment=["Wavelength ratio", "Angle", "Redshift", "Number of pairs"], header=header, - extname='ATTRI') + extname="ATTRI", + ) - header2 = [{ - 'name': 'HLPXSCHM', - 'value': 'RING', - 'comment': ' Healpix scheme' - }] + header2 = [{"name": "HLPXSCHM", "value": "RING", "comment": " Healpix scheme"}] xi_list_name = "DA" if blinding != "none": xi_list_name += "_BLIND" - results.write([healpix_list, weights_list, xi_list], - names=['HEALPID', 'WE', xi_list_name], - comment=['Healpix index', 'Sum of weight', 'Correlation'], - header=header2, - extname='COR') + results.write( + [healpix_list, weights_list, xi_list], + names=["HEALPID", "WE", xi_list_name], + comment=["Healpix index", "Sum of weight", "Correlation"], + header=header2, + extname="COR", + ) results.close() -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs) diff --git a/py/picca/bin/picca_xdmat.py b/py/picca/bin/picca_xdmat.py old mode 100644 new mode 100755 index fd08763a0..f294802ab --- a/py/picca/bin/picca_xdmat.py +++ b/py/picca/bin/picca_xdmat.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Computes the distortion matrix between of the cross-correlation delta x object @@ -7,8 +7,9 @@ """ import argparse import multiprocessing +import sys import time -from multiprocessing import Lock, Pool, Value, cpu_count +from multiprocessing import Lock, Value, cpu_count import fitsio import numpy as np @@ -108,10 +109,14 @@ def main(cmdargs=None): "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" ) - parser.add_argument('--rmu-binning', action="store_true", - help=('Estimate in r,mu binning. np becomes mu bins.' - ' nt becomes r bins. rp min max is always -1, 1') - ) + parser.add_argument( + "--rmu-binning", + action="store_true", + help=( + "Estimate in r,mu binning. np becomes mu bins." + " nt becomes r bins. rp min max is always -1, 1" + ), + ) parser.add_argument( "--coef-binning-model", @@ -558,36 +563,38 @@ def main(cmdargs=None): "value": args.fid_Or, "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", }, + {"name": "NPUSED", "value": num_pairs_used, "comment": "Number of used pairs"}, { - 'name': 'NPUSED', - 'value': num_pairs_used, - 'comment': 'Number of used pairs' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - }, { - 'name': "RMU_BIN", - 'value': xcf.rmu_binning, - 'comment': 'True if binned in r, mu' - } - ] + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", + }, + { + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + { + "name": "RMU_BIN", + "value": xcf.rmu_binning, + "comment": "True if binned in r, mu", + }, + ] dmat_name = "DM" if blinding != "none": dmat_name += "_BLIND" @@ -610,3 +617,8 @@ def main(cmdargs=None): t3 = time.time() userprint(f"picca_xdmat.py - Time total: {(t3-t0)/60:.3f} minutes") + + +if __name__ == "__main__": + cmdargs = sys.argv[1:] + main(cmdargs) diff --git a/py/picca/bin/picca_xwick.py b/py/picca/bin/picca_xwick.py old mode 100644 new mode 100755 index 877d5f97c..24e286a31 --- a/py/picca/bin/picca_xwick.py +++ b/py/picca/bin/picca_xwick.py @@ -1,17 +1,18 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Compute the wick covariance for the cross-correlation of object x forests. The wick covariance is computed as explained in Delubac et al. 2015 """ -import sys import argparse import multiprocessing -from multiprocessing import Pool, Lock, cpu_count, Value +import sys +from multiprocessing import Lock, Pool, Value, cpu_count + import fitsio import numpy as np from scipy.interpolate import interp1d -from picca import constants, io, utils, xcf, cf +from picca import cf, constants, io, utils, xcf from picca.utils import userprint @@ -40,219 +41,256 @@ def main(cmdargs): forests.""" parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=('Compute the wick covariance for the cross-correlation of ' - 'object x forests.')) - - parser.add_argument('--out', - type=str, - default=None, - required=True, - help='Output file name') - - parser.add_argument('--in-dir', - type=str, - default=None, - required=True, - help='Directory to delta files') - - parser.add_argument('--drq', - type=str, - default=None, - required=True, - help='Catalog of objects in DRQ format') + description=( + "Compute the wick covariance for the cross-correlation of " + "object x forests." + ), + ) + + parser.add_argument( + "--out", type=str, default=None, required=True, help="Output file name" + ) + + parser.add_argument( + "--in-dir", + type=str, + default=None, + required=True, + help="Directory to delta files", + ) + + parser.add_argument( + "--drq", + type=str, + default=None, + required=True, + help="Catalog of objects in DRQ format", + ) + + parser.add_argument( + "--mode", + type=str, + default="sdss", + choices=["sdss", "desi"], + required=False, + help="type of catalog supplied, default sdss", + ) + + parser.add_argument( + "--rp-min", + type=float, + default=-200.0, + required=False, + help="Min r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rp-max", + type=float, + default=200.0, + required=False, + help="Max r-parallel [h^-1 Mpc]", + ) + + parser.add_argument( + "--rt-max", + type=float, + default=200.0, + required=False, + help="Max r-transverse [h^-1 Mpc]", + ) + + parser.add_argument( + "--np", type=int, default=100, required=False, help="Number of r-parallel bins" + ) parser.add_argument( - '--mode', - type=str, - default='sdss', - choices=['sdss','desi'], - required=False, - help='type of catalog supplied, default sdss') - - parser.add_argument('--rp-min', - type=float, - default=-200., - required=False, - help='Min r-parallel [h^-1 Mpc]') - - parser.add_argument('--rp-max', - type=float, - default=200., - required=False, - help='Max r-parallel [h^-1 Mpc]') - - parser.add_argument('--rt-max', - type=float, - default=200., - required=False, - help='Max r-transverse [h^-1 Mpc]') - - parser.add_argument('--np', - type=int, - default=100, - required=False, - help='Number of r-parallel bins') - - parser.add_argument('--nt', - type=int, - default=50, - required=False, - help='Number of r-transverse bins') - - parser.add_argument('--z-min-obj', - type=float, - default=0, - required=False, - help='Min redshift for object field') - - parser.add_argument('--z-max-obj', - type=float, - default=10, - required=False, - help='Max redshift for object field') + "--nt", type=int, default=50, required=False, help="Number of r-transverse bins" + ) parser.add_argument( - '--z-cut-min', + "--z-min-obj", type=float, - default=0., + default=0, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift larger than ' - 'z-cut-min')) + help="Min redshift for object field", + ) parser.add_argument( - '--z-cut-max', + "--z-max-obj", type=float, - default=10., + default=10, required=False, - help=('Use only pairs of forest x object with the mean of the last ' - 'absorber redshift and the object redshift smaller than ' - 'z-cut-max')) + help="Max redshift for object field", + ) parser.add_argument( - '--z-min-sources', + "--z-cut-min", type=float, - default=0., + default=0.0, required=False, - help=('Limit the minimum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift larger than " + "z-cut-min" + ), + ) parser.add_argument( - '--z-max-sources', + "--z-cut-max", type=float, - default=10., + default=10.0, required=False, - help=('Limit the maximum redshift of the quasars ' - 'used as sources for spectra')) + help=( + "Use only pairs of forest x object with the mean of the last " + "absorber redshift and the object redshift smaller than " + "z-cut-max" + ), + ) parser.add_argument( - '--lambda-abs', + "--z-min-sources", + type=float, + default=0.0, + required=False, + help=( + "Limit the minimum redshift of the quasars " "used as sources for spectra" + ), + ) + + parser.add_argument( + "--z-max-sources", + type=float, + default=10.0, + required=False, + help=( + "Limit the maximum redshift of the quasars " "used as sources for spectra" + ), + ) + + parser.add_argument( + "--lambda-abs", type=str, - default='LYA', + default="LYA", required=False, - help=('Name of the absorption in picca.constants defining the redshift ' - 'of the delta')) + help=( + "Name of the absorption in picca.constants defining the redshift " + "of the delta" + ), + ) - parser.add_argument('--z-ref', - type=float, - default=2.25, - required=False, - help='Reference redshift') + parser.add_argument( + "--z-ref", type=float, default=2.25, required=False, help="Reference redshift" + ) parser.add_argument( - '--z-evol-del', + "--z-evol-del", type=float, default=2.9, required=False, - help='Exponent of the redshift evolution of the delta field') + help="Exponent of the redshift evolution of the delta field", + ) parser.add_argument( - '--z-evol-obj', + "--z-evol-obj", type=float, default=1.44, required=False, - help='Exponent of the redshift evolution of the object field') + help="Exponent of the redshift evolution of the object field", + ) parser.add_argument( - '--fid-Om', + "--fid-Om", type=float, default=0.315, required=False, - help='Omega_matter(z=0) of fiducial LambdaCDM cosmology') + help="Omega_matter(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-Or', + "--fid-Or", type=float, - default=0., + default=0.0, required=False, - help='Omega_radiation(z=0) of fiducial LambdaCDM cosmology') + help="Omega_radiation(z=0) of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--fid-Ok', - type=float, - default=0., - required=False, - help='Omega_k(z=0) of fiducial LambdaCDM cosmology') + parser.add_argument( + "--fid-Ok", + type=float, + default=0.0, + required=False, + help="Omega_k(z=0) of fiducial LambdaCDM cosmology", + ) parser.add_argument( - '--fid-wl', + "--fid-wl", type=float, - default=-1., + default=-1.0, required=False, - help='Equation of state of dark energy of fiducial LambdaCDM cosmology') + help="Equation of state of dark energy of fiducial LambdaCDM cosmology", + ) - parser.add_argument('--max-diagram', - type=int, - default=4, - required=False, - help='Maximum diagram to compute') + parser.add_argument( + "--max-diagram", + type=int, + default=4, + required=False, + help="Maximum diagram to compute", + ) parser.add_argument( - '--cf1d', + "--cf1d", type=str, required=True, - help=('1D auto-correlation of pixels from the same forest file: ' - 'picca_cf1d.py')) + help=( + "1D auto-correlation of pixels from the same forest file: " "picca_cf1d.py" + ), + ) parser.add_argument( - '--cf', + "--cf", type=str, default=None, required=False, - help=('3D auto-correlation of pixels from different forests: ' - 'picca_cf.py')) + help=("3D auto-correlation of pixels from different forests: " "picca_cf.py"), + ) parser.add_argument( - '--rej', + "--rej", type=float, - default=1., + default=1.0, + required=False, + help=( + "Fraction of rejected object-forests pairs: -1=no rejection, " + "1=all rejection" + ), + ) + + parser.add_argument( + "--nside", type=int, default=16, required=False, help="Healpix nside" + ) + + parser.add_argument( + "--nproc", type=int, default=None, required=False, help="Number of processors" + ) + + parser.add_argument( + "--nspec", + type=int, + default=None, + required=False, + help="Maximum number of spectra to read", + ) + + parser.add_argument( + "--rebin-factor", + type=int, + default=None, required=False, - help=('Fraction of rejected object-forests pairs: -1=no rejection, ' - '1=all rejection')) - - parser.add_argument('--nside', - type=int, - default=16, - required=False, - help='Healpix nside') - - parser.add_argument('--nproc', - type=int, - default=None, - required=False, - help='Number of processors') - - parser.add_argument('--nspec', - type=int, - default=None, - required=False, - help='Maximum number of spectra to read') - - parser.add_argument('--rebin-factor', - type=int, - default=None, - required=False, - help='Rebin factor for deltas. If not None, deltas will ' - 'be rebinned by that factor') + help="Rebin factor for deltas. If not None, deltas will " + "be rebinned by that factor", + ) args = parser.parse_args(cmdargs) if args.nproc is None: @@ -278,33 +316,45 @@ def main(cmdargs): blinding = io.read_blinding(args.in_dir) # load fiducial cosmology - if (args.fid_Or != 0.) or (args.fid_Ok != 0.) or (args.fid_wl != -1.): - userprint(("ERROR: Cosmology with other than Omega_m set are not yet " - "implemented")) + if (args.fid_Or != 0.0) or (args.fid_Ok != 0.0) or (args.fid_wl != -1.0): + userprint( + ("ERROR: Cosmology with other than Omega_m set are not yet " "implemented") + ) sys.exit() - cosmo = constants.Cosmo(Om=args.fid_Om, - Or=args.fid_Or, - Ok=args.fid_Ok, - wl=args.fid_wl,) + cosmo = constants.Cosmo( + Om=args.fid_Om, + Or=args.fid_Or, + Ok=args.fid_Ok, + wl=args.fid_wl, + ) ### Read deltas - data, num_data, z_min, z_max = io.read_deltas(args.in_dir, - args.nside, - xcf.lambda_abs, - args.z_evol_del, - args.z_ref, - cosmo=cosmo, - max_num_spec=args.nspec, - nproc=args.nproc, - rebin_factor=args.rebin_factor, - z_min_qso=args.z_min_sources, - z_max_qso=args.z_max_sources) + data, num_data, z_min, z_max = io.read_deltas( + args.in_dir, + args.nside, + xcf.lambda_abs, + args.z_evol_del, + args.z_ref, + cosmo=cosmo, + max_num_spec=args.nspec, + nproc=args.nproc, + rebin_factor=args.rebin_factor, + z_min_qso=args.z_min_sources, + z_max_qso=args.z_max_sources, + ) for deltas in data.values(): for delta in deltas: - delta.fname = 'D1' + delta.fname = "D1" for item in [ - 'cont', 'delta', 'order', 'ivar', 'exposures_diff', - 'mean_snr', 'mean_reso', 'mean_z', 'delta_log_lambda' + "cont", + "delta", + "order", + "ivar", + "exposures_diff", + "mean_snr", + "mean_reso", + "mean_z", + "delta_log_lambda", ]: setattr(delta, item, None) xcf.data = data @@ -316,19 +366,26 @@ def main(cmdargs): ### Find the redshift range if args.z_min_obj is None: r_comov_min = cosmo.get_r_comov(z_min) - r_comov_min = max(0., r_comov_min + xcf.r_par_min) + r_comov_min = max(0.0, r_comov_min + xcf.r_par_min) args.z_min_obj = cosmo.distance_to_redshift(r_comov_min) sys.stderr.write("\r z_min_obj = {}\r".format(args.z_min_obj)) if args.z_max_obj is None: r_comov_max = cosmo.get_r_comov(z_max) - r_comov_max = max(0., r_comov_max + xcf.r_par_max) + r_comov_max = max(0.0, r_comov_max + xcf.r_par_max) args.z_max_obj = cosmo.distance_to_redshift(r_comov_max) sys.stderr.write("\r z_max_obj = {}\r".format(args.z_max_obj)) ### Read objects - objs, z_min2 = io.read_objects(args.drq, args.nside, args.z_min_obj, - args.z_max_obj, args.z_evol_obj, args.z_ref, - cosmo, mode=args.mode) + objs, z_min2 = io.read_objects( + args.drq, + args.nside, + args.z_min_obj, + args.z_max_obj, + args.z_evol_obj, + args.z_ref, + cosmo, + mode=args.mode, + ) xcf.objs = objs sys.stderr.write("\n") userprint("done, npix = {}".format(len(objs))) @@ -340,38 +397,42 @@ def main(cmdargs): # Load 1d correlation functions hdul = fitsio.FITS(args.cf1d) header = hdul[1].read_header() - log_lambda_min = header['LLMIN'] - delta_log_lambda = header['DLL'] - num_pairs_variance_1d = hdul[1]['nv1d'][:] - variance_1d = hdul[1]['v1d'][:] + log_lambda_min = header["LLMIN"] + delta_log_lambda = header["DLL"] + num_pairs_variance_1d = hdul[1]["nv1d"][:] + variance_1d = hdul[1]["v1d"][:] log_lambda = log_lambda_min + delta_log_lambda * np.arange(variance_1d.size) - xcf.get_variance_1d['D1'] = interp1d(log_lambda[num_pairs_variance_1d > 0], - variance_1d[num_pairs_variance_1d > 0], - kind='nearest', - fill_value='extrapolate') - - num_pairs1d = hdul[1]['nb1d'][:] - xi_1d = hdul[1]['c1d'][:] - xcf.xi_1d['D1'] = interp1d((log_lambda - log_lambda_min)[num_pairs1d > 0], - xi_1d[num_pairs1d > 0], - kind='nearest', - fill_value='extrapolate') + xcf.get_variance_1d["D1"] = interp1d( + log_lambda[num_pairs_variance_1d > 0], + variance_1d[num_pairs_variance_1d > 0], + kind="nearest", + fill_value="extrapolate", + ) + + num_pairs1d = hdul[1]["nb1d"][:] + xi_1d = hdul[1]["c1d"][:] + xcf.xi_1d["D1"] = interp1d( + (log_lambda - log_lambda_min)[num_pairs1d > 0], + xi_1d[num_pairs1d > 0], + kind="nearest", + fill_value="extrapolate", + ) hdul.close() # Load correlation functions if not args.cf is None: hdul = fitsio.FITS(args.cf) header = hdul[1].read_header() - assert cf.num_bins_r_par == header['NP'] - assert cf.num_bins_r_trans == header['NT'] - assert cf.r_par_min == header['RPMIN'] - assert cf.r_par_max == header['RPMAX'] - assert cf.r_trans_max == header['RTMAX'] - xi = hdul[2]['DA'][:] - weights = hdul[2]['WE'][:] + assert cf.num_bins_r_par == header["NP"] + assert cf.num_bins_r_trans == header["NT"] + assert cf.r_par_min == header["RPMIN"] + assert cf.r_par_max == header["RPMAX"] + assert cf.r_trans_max == header["RTMAX"] + xi = hdul[2]["DA"][:] + weights = hdul[2]["WE"][:] xi = (xi * weights).sum(axis=0) weights = weights.sum(axis=0) - w = weights > 0. + w = weights > 0.0 xi[w] /= weights[w] xcf.xi_wick = xi.copy() hdul.close() @@ -383,7 +444,7 @@ def main(cmdargs): cf.z_cut_min = xcf.z_cut_min ### Send - xcf.counter = Value('i', 0) + xcf.counter = Value("i", 0) xcf.lock = Lock() cpu_data = {} @@ -400,7 +461,7 @@ def main(cmdargs): cf.fill_neighs(healpixs) # compute the covariance matrix - context = multiprocessing.get_context('fork') + context = multiprocessing.get_context("fork") pool = context.Pool(processes=min(args.nproc, len(cpu_data.values()))) userprint(" \nStarting\n") wick_data = pool.map(calc_wick_terms, sorted(cpu_data.values())) @@ -419,107 +480,108 @@ def main(cmdargs): t5 = np.array([item[8] for item in wick_data]).sum(axis=0) t6 = np.array([item[9] for item in wick_data]).sum(axis=0) weights = weights_wick * weights_wick[:, None] - w = weights > 0. + w = weights > 0.0 t1[w] /= weights[w] t2[w] /= weights[w] t3[w] /= weights[w] t4[w] /= weights[w] t5[w] /= weights[w] t6[w] /= weights[w] - t1 *= 1. * npairs_used / npairs - t2 *= 1. * npairs_used / npairs - t3 *= 1. * npairs_used / npairs - t4 *= 1. * npairs_used / npairs - t5 *= 1. * npairs_used / npairs - t6 *= 1. * npairs_used / npairs + t1 *= 1.0 * npairs_used / npairs + t2 *= 1.0 * npairs_used / npairs + t3 *= 1.0 * npairs_used / npairs + t4 *= 1.0 * npairs_used / npairs + t5 *= 1.0 * npairs_used / npairs + t6 *= 1.0 * npairs_used / npairs t_tot = t1 + t2 + t3 + t4 + t5 + t6 - results = fitsio.FITS(args.out, 'rw', clobber=True) + results = fitsio.FITS(args.out, "rw", clobber=True) header = [ { - 'name': 'RPMIN', - 'value': xcf.r_par_min, - 'comment': 'Minimum r-parallel [h^-1 Mpc]' + "name": "RPMIN", + "value": xcf.r_par_min, + "comment": "Minimum r-parallel [h^-1 Mpc]", + }, + { + "name": "RPMAX", + "value": xcf.r_par_max, + "comment": "Maximum r-parallel [h^-1 Mpc]", }, { - 'name': 'RPMAX', - 'value': xcf.r_par_max, - 'comment': 'Maximum r-parallel [h^-1 Mpc]' + "name": "RTMAX", + "value": xcf.r_trans_max, + "comment": "Maximum r-transverse [h^-1 Mpc]", }, { - 'name': 'RTMAX', - 'value': xcf.r_trans_max, - 'comment': 'Maximum r-transverse [h^-1 Mpc]' + "name": "NP", + "value": xcf.num_bins_r_par, + "comment": "Number of bins in r-parallel", }, { - 'name': 'NP', - 'value': xcf.num_bins_r_par, - 'comment': 'Number of bins in r-parallel' + "name": "NT", + "value": xcf.num_bins_r_trans, + "comment": "Number of bins in r-transverse", }, { - 'name': 'NT', - 'value': xcf.num_bins_r_trans, - 'comment': 'Number of bins in r-transverse' + "name": "ZCUTMIN", + "value": xcf.z_cut_min, + "comment": "Minimum redshift of pairs", }, { - 'name': 'ZCUTMIN', - 'value': xcf.z_cut_min, - 'comment': 'Minimum redshift of pairs' + "name": "ZCUTMAX", + "value": xcf.z_cut_max, + "comment": "Maximum redshift of pairs", }, + {"name": "REJ", "value": xcf.reject, "comment": "Rejection factor"}, + {"name": "NPALL", "value": npairs, "comment": "Number of pairs"}, + {"name": "NPUSED", "value": npairs_used, "comment": "Number of used pairs"}, { - 'name': 'ZCUTMAX', - 'value': xcf.z_cut_max, - 'comment': 'Maximum redshift of pairs' + "name": "OMEGAM", + "value": args.fid_Om, + "comment": "Omega_matter(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'REJ', - 'value': xcf.reject, - 'comment': 'Rejection factor' + "name": "OMEGAR", + "value": args.fid_Or, + "comment": "Omega_radiation(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'NPALL', - 'value': npairs, - 'comment': 'Number of pairs' + "name": "OMEGAK", + "value": args.fid_Ok, + "comment": "Omega_k(z=0) of fiducial LambdaCDM cosmology", }, { - 'name': 'NPUSED', - 'value': npairs_used, - 'comment': 'Number of used pairs' - }, { - 'name': 'OMEGAM', - 'value': args.fid_Om, - 'comment': 'Omega_matter(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAR', - 'value': args.fid_Or, - 'comment': 'Omega_radiation(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'OMEGAK', - 'value': args.fid_Ok, - 'comment': 'Omega_k(z=0) of fiducial LambdaCDM cosmology' - }, { - 'name': 'WL', - 'value': args.fid_wl, - 'comment': 'Equation of state of dark energy of fiducial LambdaCDM cosmology' - }, { - 'name': "BLINDING", - 'value': blinding, - 'comment': 'String specifying the blinding strategy' - } - ] + "name": "WL", + "value": args.fid_wl, + "comment": "Equation of state of dark energy of fiducial LambdaCDM cosmology", + }, + { + "name": "BLINDING", + "value": blinding, + "comment": "String specifying the blinding strategy", + }, + ] comment = [ - 'Sum of weight', 'Covariance', 'Nomber of pairs', 'T1', 'T2', 'T3', - 'T4', 'T5', 'T6' + "Sum of weight", + "Covariance", + "Nomber of pairs", + "T1", + "T2", + "T3", + "T4", + "T5", + "T6", ] results.write( [t_tot, weights_wick, num_pairs_wick, t1, t2, t3, t4, t5, t6], - names=['CO', 'WALL', 'NB', 'T1', 'T2', 'T3', 'T4', 'T5', 'T6'], + names=["CO", "WALL", "NB", "T1", "T2", "T3", "T4", "T5", "T6"], comment=comment, header=header, - extname='COV') + extname="COV", + ) results.close() -if __name__ == '__main__': - cmdargs=sys.argv[1:] +if __name__ == "__main__": + cmdargs = sys.argv[1:] main(cmdargs)