uproot can not open files on dCache when 'https' protocol is used. #1255
Description
When I tried to open a file on dCache using 'https'+X509, uproot fails to open it. I am using:
Python 3.12.4 | packaged by Anaconda, Inc. | (main, Jun 18 2024, 15:12:24) [GCC 11.2.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import uproot
>>> uproot.__version__
'5.3.10'
This can be reproduced with :
import sys
import os
import ssl
import uproot
filenames = [{"T1_US_FNAL root":"root://cmsxrootd-site2.fnal.gov//store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root"}]
filenames.append({"T2_US_Wisconsin https":"https://cmsxrootd.hep.wisc.edu:1094/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/44187D37-0301-3942-A6F7-C723E9F4813D.root"})
filenames.append({"T1_US_FNAL https":"https://cmsdcadisk.fnal.gov:2880/dcache/uscmsdisk/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root"})
filenames.append({"T2_DE_DESY https":"https://dcache-cms-webdav-wan.desy.de:2880//pnfs/desy.de/cms/tier2/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root"})
i=0
for thefile in filenames:
i=i+1
try:
sslctx = ssl.create_default_context()
sslctx.load_cert_chain(os.environ["X509_USER_PROXY"], os.environ["X509_USER_PROXY"])
uproot_options={'ssl': sslctx}
site_protocol=list(thefile.keys())[0]
the_file = uproot.open({thefile[site_protocol]: None}, **uproot_options)
print ("[",i,"] OPEN OK ",site_protocol, thefile[site_protocol], " size of CA certs ", len(uproot_options['ssl'].get_ca_certs()))
#print ("[",i,"] file is open and the_file is ",the_file)
the_file.close()
except Exception as e:
print ( "[",i,"] OPEN Exception ",site_protocol, thefile[site_protocol], " Exception was ",e)
The output of the above script looks like:
[ 1 ] OPEN OK T1_US_FNAL root root://cmsxrootd-site2.fnal.gov//store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root size of CA certs 147
[ 2 ] OPEN OK T2_US_Wisconsin https https://cmsxrootd.hep.wisc.edu:1094/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/44187D37-0301-3942-A6F7-C723E9F4813D.root size of CA certs 147
[ 3 ] OPEN Exception T1_US_FNAL https https://cmsdcadisk.fnal.gov:2880/dcache/uscmsdisk/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root Exception was https://cmsdcadisk.fnal.gov:2880/dcache/uscmsdisk/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root
[ 4 ] OPEN Exception T2_DE_DESY https https://dcache-cms-webdav-wan.desy.de:2880//pnfs/desy.de/cms/tier2/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root Exception was https://dcache-cms-webdav-wan.desy.de:2880//pnfs/desy.de/cms/tier2/store/mc/RunIISummer20UL18NanoAODv9/TTTo2L2Nu_TuneCP5_13TeV-powheg-pythia8/NANOAODSIM/106X_upgrade2018_realistic_v16_L1v1-v1/130000/5D2E4672-C7D3-AF49-B699-E0F7E83A699C.root
To access the files, one needs to pass X509 using SSLContext like above and might need to add the SSLContext like so:
lib/python3.12/site-packages/fsspec/implementations/http.py : add the creation of the SSLContext around lines between 224 and 225 and between 825 and 826 like so:
import os
import ssl
import socket
import copyreg
def save_sslcontext(obj):
return obj.__class__, (obj.protocol,)
copyreg.pickle(ssl.SSLContext, save_sslcontext)
sslctx = ssl.create_default_context()
sslctx.load_cert_chain(os.environ['X509_USER_PROXY'], os.environ['X509_USER_PROXY'])
sslctxdic={'ssl': sslctx}
# Last - 0 necessary
kw.update(sslctxdic)