From 790523dd0511124d1e9400e15dc83b66e25e08dd Mon Sep 17 00:00:00 2001 From: cabuzth Date: Mon, 2 Oct 2023 19:48:22 +0200 Subject: [PATCH 1/8] Ensure folders exist when writing to them --- newspaper/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/newspaper/utils.py b/newspaper/utils.py index bfa44148..1f2fbe0c 100644 --- a/newspaper/utils.py +++ b/newspaper/utils.py @@ -235,6 +235,7 @@ def inner_function(*args, **kwargs): # call the decorated function... result = function(*args, **kwargs) # ... and save the cached object for next time + os.makedirs(cache_folder, exist_ok=True) pickle.dump(result, open(filepath, "wb")) return result return inner_function @@ -324,6 +325,7 @@ def memoize_articles(source, articles): memo_text = '' # TODO if source: source.write_upload_times(prev_length, new_length) + os.makedirs(settings.MEMO_DIR, exist_ok=True) ff = codecs.open(d_pth, 'w', 'utf-8') ff.write(memo_text) ff.close() From f10422585b25634c4eda82e2ce0ccc3d3b6055c8 Mon Sep 17 00:00:00 2001 From: cabuzth Date: Mon, 2 Oct 2023 19:49:32 +0200 Subject: [PATCH 2/8] Support Session object for allowing login before building --- newspaper/configuration.py | 2 ++ newspaper/network.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 94688e70..7ef50845 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -11,6 +11,7 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' import logging +import requests from .parsers import Parser from .text import (StopWords, StopWordsArabic, StopWordsChinese, @@ -63,6 +64,7 @@ def __init__(self): # Unique stopword classes for oriental languages, don't toggle self.stopwords_class = StopWords + self.session = requests.Session() self.browser_user_agent = 'newspaper/%s' % __version__ self.headers = {} self.request_timeout = 7 diff --git a/newspaper/network.py b/newspaper/network.py index 29f0e699..c91706fa 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -55,11 +55,12 @@ def get_html_2XX_only(url, config=None, response=None): timeout = config.request_timeout proxies = config.proxies headers = config.headers + session = config.session if response is not None: return _get_html_from_response(response, config) - response = requests.get( + response = session.get( url=url, **get_request_kwargs(timeout, useragent, proxies, headers)) html = _get_html_from_response(response, config) @@ -102,11 +103,12 @@ def __init__(self, url, config=None): self.timeout = config.request_timeout self.proxies = config.proxies self.headers = config.headers + self.session = config.session self.resp = None def send(self): try: - self.resp = requests.get(self.url, **get_request_kwargs( + self.resp = self.session.get(self.url, **get_request_kwargs( self.timeout, self.useragent, self.proxies, self.headers)) if self.config.http_success_only: self.resp.raise_for_status() From afea0880cb5e76fa290dda5e13582f5f4a3e3414 Mon Sep 17 00:00:00 2001 From: cabuzth Date: Tue, 3 Oct 2023 14:21:21 +0200 Subject: [PATCH 3/8] Allow for shorter and numeric article urls --- newspaper/urls.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newspaper/urls.py b/newspaper/urls.py index 2126aaf3..bee1e1e0 100644 --- a/newspaper/urls.py +++ b/newspaper/urls.py @@ -211,6 +211,11 @@ def valid_url(url, verbose=False, test=False): if verbose: print('%s verified for being a slug' % url) return True + # Allow for paths like /[numeric] (eg: https://carbon-pulse.com/226570/) + if len(path_chunks) == 1 and path_chunks[0].isnumeric(): + if verbose: print('%s verified for isnumeric' % url) + return True + # There must be at least 2 subpaths if len(path_chunks) <= 1: if verbose: print('%s caught for path chunks too small' % url) From afb6ed8e5ffab8e57a1367d6c20141a84cd9963b Mon Sep 17 00:00:00 2001 From: cabuzth Date: Tue, 3 Oct 2023 14:22:02 +0200 Subject: [PATCH 4/8] Some more stopwords --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 96255401..1bc97157 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -709,7 +709,7 @@ def get_category_urls(self, source_url, doc): 'tickets', 'coupons', 'forum', 'board', 'archive', 'browse', 'howto', 'how to', 'faq', 'terms', 'charts', 'services', 'contact', 'plus', 'admin', 'login', 'signup', 'register', - 'developer', 'proxy'] + 'developer', 'proxy', 'what-we-offer', 'staff'] _valid_categories = [] From 16001a9eca5f98b255adaa958a6eeff264121bee Mon Sep 17 00:00:00 2001 From: cabuzth Date: Tue, 3 Oct 2023 14:23:20 +0200 Subject: [PATCH 5/8] Removing some duplicated early --- newspaper/extractors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 1bc97157..09e25fcf 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -711,6 +711,7 @@ def get_category_urls(self, source_url, doc): 'contact', 'plus', 'admin', 'login', 'signup', 'register', 'developer', 'proxy', 'what-we-offer', 'staff'] + valid_categories = list(set(valid_categories)) _valid_categories = [] # TODO Stop spamming urlparse and tldextract calls... From 06fd9a2d135f1ea5b78fb4b87b79fc88bc35ce57 Mon Sep 17 00:00:00 2001 From: cabuzth Date: Tue, 3 Oct 2023 14:31:58 +0200 Subject: [PATCH 6/8] Allow some more category url Add '//' before domain to correct a bug generating category_urls in the form of scheme://domain.tld/domain.tdl/path Don't do Domain + Path when domain is '' --- newspaper/extractors.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 09e25fcf..4a2d955c 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -681,18 +681,29 @@ def get_category_urls(self, source_url, doc): 'subdomain' % p_url)) continue else: - valid_categories.append(scheme + '://' + domain) - # TODO account for case where category is in form - # http://subdomain.domain.tld/category/ <-- still legal! + if subdomain_contains: + valid_categories.append(scheme + '://' + domain) + # TODO account for case where category is in form + # http://subdomain.domain.tld/category/ <-- still legal! + else: + # support for urls like + # http://domain.tld/category/[category] + path_chunks = [x for x in path.split('/') if len(x) > 0] + path_chunks = [x for x in path_chunks if x not in set(['index.html', 'category'])] + path_chunks = [x for x in path_chunks if not x.isnumeric()] + + if len(path_chunks) == 1 and len(path_chunks[0]) < 14: + valid_categories.append('//' + domain + path) else: # we want a path with just one subdir # cnn.com/world and cnn.com/world/ are both valid_categories + # carbon-pulse.com/category/international/ is a valid_categories path_chunks = [x for x in path.split('/') if len(x) > 0] - if 'index.html' in path_chunks: - path_chunks.remove('index.html') + path_chunks = [x for x in path_chunks if x not in set(['index.html', 'category'])] + path_chunks = [x for x in path_chunks if not x.isnumeric()] if len(path_chunks) == 1 and len(path_chunks[0]) < 14: - valid_categories.append(domain + path) + valid_categories.append(path) else: if self.config.verbose: print(('elim category url %s for >1 path chunks ' From 465be4d54411a46f52c441fef0cf0414bf352ca3 Mon Sep 17 00:00:00 2001 From: cabuzth Date: Tue, 3 Oct 2023 14:53:03 +0200 Subject: [PATCH 7/8] Code beautification --- newspaper/extractors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 4a2d955c..1e472978 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -759,8 +759,7 @@ def get_category_urls(self, source_url, doc): _valid_categories = list(set(_valid_categories)) - category_urls = [urls.prepare_url(p_url, source_url) - for p_url in _valid_categories] + category_urls = [urls.prepare_url(p_url, source_url) for p_url in _valid_categories] category_urls = [c for c in category_urls if c is not None] return category_urls From 4e49d71d32f68bf4f09208c410d22ddcb6ed2293 Mon Sep 17 00:00:00 2001 From: cabuzth Date: Thu, 29 Aug 2024 15:17:02 +0200 Subject: [PATCH 8/8] requirements.txt update --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 61974601..4348fccf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,11 @@ feedfinder2>=0.0.4 feedparser>=5.2.1 jieba3k>=0.35.1 lxml>=3.6.0 -nltk>=3.2.1 -Pillow>=3.3.0 +nltk>=3.6.6 +Pillow>=10.0.1 pythainlp>=1.7.2 python-dateutil>=2.5.3 PyYAML>=3.11 requests>=2.10.0 -tinysegmenter==0.3 # TODO(codelucas): Investigate making this >=0.3 +tinysegmenter>=0.4 tldextract>=2.0.1 \ No newline at end of file