Skip to content

Commit

Permalink
Fix issue with OECD server not supporting RFC 5746
Browse files Browse the repository at this point in the history
  • Loading branch information
markbrough committed Jul 13, 2023
1 parent 7ea177c commit 65efb0e
Showing 1 changed file with 27 additions and 1 deletion.
28 changes: 27 additions & 1 deletion scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,35 @@
CRS_CODES_URL = "https://stats.oecd.org/ModalDimWebTree.aspx?DimensionCode=SECTOR&SubSessionId={}&Random=0.2017416214402572"


# from https://stackoverflow.com/a/73519818

import urllib3
import ssl

class CustomHttpAdapter (requests.adapters.HTTPAdapter):
# "Transport adapter" that allows us to use custom ssl_context.

def __init__(self, ssl_context=None, **kwargs):
self.ssl_context = ssl_context
super().__init__(**kwargs)

def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = urllib3.poolmanager.PoolManager(
num_pools=connections, maxsize=maxsize,
block=block, ssl_context=self.ssl_context)


def get_legacy_session():
ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
ctx.options |= 0x4 # OP_LEGACY_SERVER_CONNECT
session = requests.session()
session.mount('https://', CustomHttpAdapter(ctx))
return session


class CodesGetter():
def get_doc(self):
s = requests.Session()
s = get_legacy_session()
s.headers.update({'User-Agent': 'Mozilla/5.0'})
s.get(OECD_STATS_URL)
req_crs = s.get(CRS_URL.format(self.lang))
Expand Down

0 comments on commit 65efb0e

Please sign in to comment.