diff --git a/Extraction/SalesforceCustomExtractionUtility/README.md b/Extraction/SalesforceCustomExtractionUtility/README.md index d8312b89..063d2f85 100644 --- a/Extraction/SalesforceCustomExtractionUtility/README.md +++ b/Extraction/SalesforceCustomExtractionUtility/README.md @@ -74,16 +74,17 @@ Follow the steps below to set up and run the Salesforce Custom Extraction Utilit #Proxy URLs for network access proxies= {"http": "PROXY URL","https": "PROXY URL"} - #Specifies whether data fetching uses URL names or Item IDs - #inputFormat="urlnames" or inputFormat="itemids" - inputFormat ="" + + #input type = "articleId" [If we need to fetch data from selected article ids] or "urlNames" [If we need to fetch data using csv file containing UrlNames] or ''[empty incase of fetching the whole data] + inputType="articleId" + + #itemIds format incase of selected article ids ['kA0J1000000oLlhKAE','kA0J1000000oLlOKAU','kA0J1000000oM5LKAU'] or '' incase the article ids are mentioned in csv file + itemIds='' #SSL Verification #Set to "False" if SSL Verification is not needed, otherwise set to "True" ssl="" - #If you need the details of specific documents you can enter their article ids or leave it as empty - itemIds="" ``` diff --git a/Extraction/SalesforceCustomExtractionUtility/emptyUrl.csv b/Extraction/SalesforceCustomExtractionUtility/emptyUrl.csv new file mode 100644 index 00000000..e69de29b diff --git a/Extraction/SalesforceCustomExtractionUtility/input.csv b/Extraction/SalesforceCustomExtractionUtility/input.csv new file mode 100644 index 00000000..e69de29b diff --git a/Extraction/SalesforceCustomExtractionUtility/requirements.txt b/Extraction/SalesforceCustomExtractionUtility/requirements.txt index f5e5db3a..3948201a 100644 --- a/Extraction/SalesforceCustomExtractionUtility/requirements.txt +++ b/Extraction/SalesforceCustomExtractionUtility/requirements.txt @@ -7,4 +7,5 @@ urllib3==1.26.15 python-dotenv requests pandas -aiohttp \ No newline at end of file +aiohttp +ast \ No newline at end of file diff --git a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py index ed19a1a2..24ba3304 100644 --- a/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py +++ b/Extraction/SalesforceCustomExtractionUtility/salesforceData_extraction.py @@ -7,6 +7,7 @@ import ssl import shutil import csv +import ast from utils.logger import get_logger, write_json_to_separate_file,write_refresh_token logger = get_logger() #access token generation @@ -75,86 +76,91 @@ def make_api_call(url, headers=None): #getting id async def get_articles_list_id(access_token): try: - count=0 - if config.inputFormat=="urlnames": - urlNames=[] - item_ids=[] - emptyUrlNames=[] - with open('input.csv', 'r') as csvfile: - csvreader = csv.reader(csvfile) - for row in csvreader: - # print(row) - urlNames.append(row[0]) - print(len(urlNames)) - #splitting into batches - urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] - # print(urlBatches) - for eachBatch in urlBatches: - count=count+1; - logger.info(f"API calls for fetching item ids are successful for batch {count} .") - for url_name in eachBatch: - - # print(url_name) - lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" - # print(lists_url) - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US", - } - listResponse = make_api_call(lists_url,headers) - if listResponse['totalSize']==0 or listResponse['records'] == None : - emptyUrlNames.append(url_name) - - else : - itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])] - # print(itemId) - item_ids.append(itemId[0]) - logger.info(f"API calls for fetching item ids are successful for batch {count} .") - # print(f"API calls for fetching item ids are successful for batch {count} .") - emptyUrlNameslist = [[item] for item in emptyUrlNames] - print("Empty URLS",emptyUrlNameslist) + urlNames=[] + item_ids=[] + emptyUrlNames=[] + ItemIds=config.itemIds + # print(expectedItemIds) + if config.inputType=='': + lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US", + } - with open('emptyUrl.csv', 'w', newline='') as csvfile: - csvwriter = csv.writer(csvfile) - csvwriter.writerows(emptyUrlNameslist) - elif config.inputFormat=="itemids": - expectedItemIds=config.itemIds - if expectedItemIds!="" : - item_ids= expectedItemIds - else: - lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId FROM KnowledgeArticleVersion WHERE PublishStatus='Online'" - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US", - } - lists_response = make_api_call(lists_url, headers) - # print("lists_response",lists_response) - item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] - list_url_all=[] - # print(item_ids) - - #getting all the other page urls and the article ids in them - if 'nextRecordsUrl' in lists_response: - for page in lists_response['nextRecordsUrl']: - if lists_response['nextRecordsUrl']: - headers = { - "Authorization": f"Bearer {access_token}", - "Accept-Language": "en-US" - } - lists_url_2 = f"{config.hostUrl}{lists_response['nextRecordsUrl']}" - lists_responses = make_api_call(lists_url_2, headers) - item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] - - item_ids=item_ids + item_id_all - lists_response=lists_responses - list_url_all.append(lists_url_2) - - + lists_response = make_api_call(lists_url, headers) + item_ids = [item["KnowledgeArticleId"] for item in lists_response.get("records", [])] + list_url_all=[] + # print(item_ids) + + #getting all the other page urls and the article ids in them + if 'nextRecordsUrl' in lists_response: + for page in lists_response['nextRecordsUrl']: + if lists_response['nextRecordsUrl']: + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US" + } + lists_url_2 = f"{config.hostUrl}{lists_response['nextRecordsUrl']}" + lists_responses = make_api_call(lists_url_2, headers) + item_id_all = [item["KnowledgeArticleId"] for item in lists_responses.get("records", [])] + + item_ids=item_ids + item_id_all + lists_response=lists_responses + list_url_all.append(lists_url_2) + + elif config.inputType=='articleId' and ItemIds!="": + expectedItemIds = ast.literal_eval(ItemIds) + item_ids= expectedItemIds + elif config.inputType=='articleId' and ItemIds=="": + + with open('input.csv', 'r') as csvfile: + csvreader = csv.reader(csvfile) + for row in csvreader: + urlNames.append(row[0]) + print(len(urlNames)) + item_ids=urlNames + + elif config.inputType=='urlNames': + with open('input.csv', 'r') as csvfile: + csvreader = csv.reader(csvfile) + for row in csvreader: + urlNames.append(row[0]) + print(len(urlNames)) + + urlBatches = [urlNames[i:i+int(config.eachBatchCount)] for i in range(0, len(urlNames), int(config.eachBatchCount))] + count=0 + for eachBatch in urlBatches: + count=count+1; + logger.info(f"API calls for fetching item ids are successful for batch {count} .") + for url_name in eachBatch: + lists_url = f"{config.hostUrl}/services/data/v57.0/query/?q=SELECT Id, Title, LastModifiedDate, KnowledgeArticleId, URLName FROM KnowledgeArticleVersion WHERE PublishStatus='Online' AND UrlName='{url_name}'" + # print(lists_url) + headers = { + "Authorization": f"Bearer {access_token}", + "Accept-Language": "en-US", + } + listResponse = make_api_call(lists_url,headers) + if listResponse['totalSize']==0 or listResponse['records'] == None : + emptyUrlNames.append(url_name) + + else : + itemId = [item["KnowledgeArticleId"] for item in listResponse.get("records", [])] + # print(itemId) + item_ids.append(itemId[0]) + # print(f"API calls for fetching item ids are successful for batch {count} .") + emptyUrlNameslist = [[item] for item in emptyUrlNames] + print("Empty URLS",emptyUrlNameslist) + + with open('emptyUrl.csv', 'w', newline='') as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerows(emptyUrlNameslist) + # print(item_ids) except Exception as e: logger.error(f"An error occurred while retrieving 'Articles' list ID: {e}", exc_info=True) logger.error(f"API calls for fetching item ids failed at batch {count} .") - + return item_ids #getting details async def fetch_item_details(new_access_token,instance_url,item_id,proxy_url): @@ -224,6 +230,7 @@ async def make_list_api_call(access_token, instance_url,itemIds,refresh_token,pr #salesforce data extraction async def extractData(): try: + access_token,instance_url,refresh_token = generate_access_token() logger.info("Access token generated successfully.") logger.debug(f"Access token: {access_token}") diff --git a/Extraction/SalesforceCustomExtractionUtility/variables.py b/Extraction/SalesforceCustomExtractionUtility/variables.py index 31aaf39a..8e513ac6 100644 --- a/Extraction/SalesforceCustomExtractionUtility/variables.py +++ b/Extraction/SalesforceCustomExtractionUtility/variables.py @@ -49,13 +49,14 @@ #proxies proxies=os.environ.get('proxies') - -#Fetching definiton -inputFormat=os.environ.get('inputFormat') - #set SSL verfication sslVerify=os.environ.get('ssl') +#input type +inputType=os.environ.get('inputType') + #itemIds itemIds=os.environ.get('itemIds') + +