Bring current, turning off logs.

Daniel Tharp · Daniel Tharp · commit 3c313d29bd15 · 2020-02-27T10:30:34.000-07:00
diff --git a/2stacks-get-forum-categories.py b/2stacks-get-forum-categories.py
@@ -9,9 +9,9 @@
 import os
 
 #temp
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 def lambda_handler(event, context):
     for record in event['Records']:
@@ -57,6 +57,6 @@ def lambda_handler(event, context):
         #  Send everything to SCUTTLE
         headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
         r = requests.put(callback_url + '/2stacks/forum/metadata', data=output, headers=headers)
-        if r.status_code == 500:
-            logger.info('500:')
-            logger.info(r.text)
+        # if r.status_code == 500:
+        #     logger.info('500:')
+        #     logger.info(r.text)
diff --git a/2stacks-get-forum-threads.py b/2stacks-get-forum-threads.py
@@ -4,16 +4,16 @@
 import config
 import helpers
 
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 def lambda_handler(event, context):
     for record in event['Records']:
         callback_url = record['messageAttributes']['callback_url']['stringValue']
         forum_id = record['messageAttributes']['forum_id']['stringValue']
         wikidot_site = record['messageAttributes']['wikidot_site']['stringValue']
-        logger.info('Fetching forum ' + forum_id + ' for ' + wikidot_site)
+        # logger.info('Fetching forum ' + forum_id + ' for ' + wikidot_site)
         
         page_no = 1
         
@@ -35,7 +35,7 @@ def lambda_handler(event, context):
                 pages = re.findall('(?:<span class="pager-no">페이지: 1 / )(\d*)', haystack) # This technically returns 2 indistinguishable objects because Wikidot.
             else:  # SCP-EN and English-speaking wikis (Some -INT sites didn't have this translated, like -RU, -UA, -CN...)
                 pages = re.findall('(?:<span class="pager-no">page 1 of )(\d*)', haystack) # This technically returns 2 indistinguishable objects because Wikidot.
-            logger.info('There are ' + str(pages) + ' pages of threads to look through.')
+            # logger.info('There are ' + str(pages) + ' pages of threads to look through.')
         except:  # This only really fails on a deleted page.
             # TODO Make scuttle handle this.
             return False
@@ -62,7 +62,7 @@ def lambda_handler(event, context):
                     return False
                 payload = {"wd_forum_id": forum_id, "threads": threads}
                 output = json.dumps(payload)
-                logger.info('Sending page ' + str(page_no) + ' to SCUTTLE')
+                # logger.info('Sending page ' + str(page_no) + ' to SCUTTLE')
                 #  Send everything to SCUTTLE
                 headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
                 r = requests.put(callback_url + '/2stacks/forum/threads', data=output, headers=headers)
diff --git a/2stacks-get-page-files.py b/2stacks-get-page-files.py
@@ -9,9 +9,9 @@
 from time import sleep
 from xmlrpc.client import ServerProxy
 
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 def lambda_handler(event, context):
     for record in event['Records']:
@@ -60,7 +60,7 @@ def lambda_handler(event, context):
                     else:
                         pass
             for idx, file in enumerate(files):
-                logger.info(idx)
+                # logger.info(idx)
                 sanitizedfilename = urllib.parse.quote(file, safe='')
                 info = s.files.get_meta({"site": wikidot_site, "page": slug, "files": [file]})
                 # logger.info(info[file]['download_url'])
@@ -85,9 +85,9 @@ def lambda_handler(event, context):
                 headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
                 j = json.dumps(payload)
                 r = requests.put(callback_url + '/2stacks/page/files', data=j, headers=headers)
-                if r.status_code == 500:
-                    logger.info('500:')
-                    logger.info(r.text)
+                # if r.status_code == 500:
+                #     logger.info('500:')
+                #     logger.info(r.text)
                 #  Be Nice
                 sleep(0.25)
     
diff --git a/2stacks-get-page-metadata.py b/2stacks-get-page-metadata.py
@@ -5,47 +5,61 @@
 import config
 import helpers
 import requests
+import xmlrpc.client
 from time import sleep
 from xmlrpc.client import ServerProxy
 
 #temp
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 def lambda_handler(event, context):
     for record in event['Records']:
         callback_url = record['messageAttributes']['callback_url']['stringValue']
         slug = record['messageAttributes']['page_slug']['stringValue']
         wikidot_site = record['messageAttributes']['wikidot_site']['stringValue']
-        logger.info('get-page-metadata for ' + slug)
+        # logger.info('get-page-metadata for ' + slug)
         #  Hit Wikidot's API
         s = ServerProxy('https://' + config.wikidot_username + ':' + config.wikidot_api_key + '@www.wikidot.com/xml-rpc-api.php')
         try:
             wp = s.pages.get_one({'site': wikidot_site, 'page': slug})
-        except:
-            # Page has been deleted
-            return False # Clean this up later.
+        except xmlrpc.client.Fault as err:
+            if err.faultCode == 403:
+                # Certain pages can be forbidden from access via the API. Yes, this is dumb.
+                # We'll get the page ID and send that to SCUTTLE
+                wdpage = requests.get('http://' + wikidot_site + ".wikidot.com/" + slug + "/norender/true")
+                # logger.info(wdpage)
+                wd_page_id = re.search("(?:WIKIREQUEST.info.pageId = )([^;]*)", wdpage.text).group(1)
+                payload = json.dumps({"slug": slug, "wd_page_id": wd_page_id, "api_status": 403})
+                # Let SCUTTLE know we're not gonna get everything.
+                # logger.info('Page protected 403, ending early, making SCUTTLE request at ' + callback_url)
+                headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
+                r = requests.put(callback_url + '/2stacks/page/metadata', data=payload, headers=headers)
+                return { "job": "complete-403" }
+                # logger.info(r.text)
+                # return r.text
         
         #  Be Nice
         sleep(0.25)
         
         #  Hit Wikidot's Frontend
-        wdpage = requests.get('http://' + wikidot_site + ".wikidot.com/" + slug)
-        logger.info(wdpage)
+        wdpage = requests.get('http://' + wikidot_site + ".wikidot.com/" + slug + "/norender/true")
+        # logger.info(wdpage.text)
         
         try:
             wd_page_id = re.search("(?:WIKIREQUEST.info.pageId = )([^;]*)", wdpage.text).group(1)
-            logger.info(wd_page_id)
+        # logger.info(wd_page_id)
         except AttributeError: # The page we got from Wikidot doesn't have a page ID.
         # This can happen if they're returning a 5XX error, or the page has been deleted.
         # TODO: Look at wdpage for a 200 status to better determine why it's crapping out.
             for i in range(5):
                 sleep(1) # Give wikidot a chance.
                 try:
-                    wd_page_id = re.search("(?:WIKIREQUEST.info.pageId = )([^;]*)", wdpage).group(1)
+                    wd_page_id = re.search("(?:WIKIREQUEST.info.pageId = )([^;]*)", wdpage.text).group(1)
                 except AttributeError:
-                    continue
+                    # It gone.
+                    return {"Slug": slug, "status": "No_page_ID"}
                 else:
                     pass
             
@@ -64,9 +78,9 @@ def lambda_handler(event, context):
         #  Send everything to SCUTTLE
         headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
         j = json.dumps(payload)
-        logger.info('Making SCUTTLE request at ' + callback_url)
+        # logger.info('Making SCUTTLE request at ' + callback_url)
         r = requests.put(callback_url + '/2stacks/page/metadata', data=j, headers=headers)
-        logger.info(r.text)
+        # logger.info(r.text)
     return {
         'job': 'complete'
     }
diff --git a/2stacks-get-page-revisions.py b/2stacks-get-page-revisions.py
@@ -9,18 +9,18 @@
 import os
 
 #temp
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 
 def lambda_handler(event, context):
     for record in event['Records']:
         callback_url = record['messageAttributes']['callback_url']['stringValue']
         wd_page_id = record['messageAttributes']['page_id']['stringValue']
         wikidot_site = record['messageAttributes']['wikidot_site']['stringValue']
-        logger.info(wikidot_site)
-        logger.info(wd_page_id)
+        # logger.info(wikidot_site)
+        # logger.info(wd_page_id)
 
         data = {'page_id': wd_page_id, 'moduleName': 'history/PageRevisionListModule', 'perpage': 99999}
         haystack = helpers.fetch(data, wikidot_site)
@@ -57,10 +57,10 @@ def lambda_handler(event, context):
             revision_type[idx] = ''.join(revision) # Flatten the tuple to one string object.
         
         innerpayload = {}
-        logger.info(str(len(revision_ids)) + " revisions.")
-        logger.info(str(len(revision_type)) + " revision type rows.")
-        for row in range(len(revision_type)):
-            logger.info(revision_type[row])
+        # logger.info(str(len(revision_ids)) + " revisions.")
+        # logger.info(str(len(revision_type)) + " revision type rows.")
+        # for row in range(len(revision_type)):
+        #     logger.info(revision_type[row])
         for row in range(len(revision_ids)):
             # logger.info("Processing revision " + revision_numbers[row])
             # We need to handle some edge cases for deleted and anonymous users.
@@ -85,9 +85,9 @@ def lambda_handler(event, context):
         #  Send everything to SCUTTLE
         headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
         r = requests.put(callback_url + '/2stacks/page/revisions', data=output, headers=headers)
-        if r.status_code == 500:
-            logger.info('500:')
-            logger.info(r.text)
+        # if r.status_code == 500:
+        #     logger.info('500:')
+        #     logger.info(r.text)
 
     return {
         'job': 'complete'
diff --git a/2stacks-get-page-thread-id.py b/2stacks-get-page-thread-id.py
@@ -9,9 +9,9 @@
 import os
 
 #temp
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 
 def lambda_handler(event, context):
@@ -22,7 +22,7 @@ def lambda_handler(event, context):
         
         data = {'pageId': wd_page_id, 'moduleName': 'forum/ForumCommentsListModule'}
         haystack = helpers.fetch(data, wikidot_site)
-        logger.info(haystack)
+        # logger.info(haystack)
         try:
             thread_id = re.search('(?:forumThreadId = )(\d*)', haystack).group(1)
         except:  # This only really fails on a deleted page.
diff --git a/2stacks-get-revision-content.py b/2stacks-get-revision-content.py
@@ -9,33 +9,33 @@
 import os
 
 #temp
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 
 def lambda_handler(event, context):
     for record in event['Records']:
-        logger.info(record['messageAttributes'])
+        # logger.info(record['messageAttributes'])
         callback_url = record['messageAttributes']['callback_url']['stringValue']
         wd_revision_id = record['messageAttributes']['revision_id']['stringValue']
         wikidot_site = record['messageAttributes']['wikidot_site']['stringValue']
         wd_url = record['messageAttributes']['wikidot_url']['stringValue']
         
-        logger.info(wd_revision_id)
-        logger.info(wikidot_site)
+        # logger.info(wd_revision_id)
+        # logger.info(wikidot_site)
         
         data = {'revision_id': wd_revision_id, 'moduleName': 'history/PageSourceModule'}
         haystack = helpers.fetch(data, wd_url)
         if haystack is None:
             return { 'revision': 'deleted ' }
         else:
-            logger.info('haystack:')
-            logger.info(haystack)
+            # logger.info('haystack:')
+            # logger.info(haystack)
             content = re.search('(?:<div class="page-source">)(.*)(?:<\/div>$)', haystack, re.DOTALL).group(1)
             payload = {"wd_revision_id": str(wd_revision_id), "content": content}
             output = json.dumps(payload)
-            logger.info("got output")
+            # logger.info("got output")
             #  Send everything to SCUTTLE
             headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
             r = requests.put(callback_url + '/2stacks/revision/content', data=output, headers=headers)
diff --git a/2stacks-get-thread-posts.py b/2stacks-get-thread-posts.py
@@ -10,9 +10,9 @@
 from bs4 import BeautifulSoup
 
 #temp
-import logging
-logger = logging.getLogger()
-logger.setLevel(logging.INFO)
+# import logging
+# logger = logging.getLogger()
+# logger.setLevel(logging.INFO)
 
 
 def lambda_handler(event, context):
@@ -54,7 +54,7 @@ def lambda_handler(event, context):
         
         # Get the OP of the thread. This is Wikidot for a per-page discussion thread or a user id otherwise.
         attribution = descriptionblock.find("span", {"class": "printuser"})
-        logger.info(attribution)
+        # logger.info(attribution)
         if attribution.string == "Wikidot":
             op_user_id = 0
             op_username = "Wikidot"
@@ -99,7 +99,7 @@ def lambda_handler(event, context):
         # Let's handle things the same way for one page or many.
         for page in range(maxpages):
             actualpage = page + 1
-            logger.info('On Page ' + str(actualpage))
+            # logger.info('On Page ' + str(actualpage))
             innerpayload = {}
             haystack = get_thread_page(thread=wd_thread_id, page=actualpage, wikidot_site=wikidot_site)  # I'm too lazy to not just increment this range by one to make it work.
             soup = BeautifulSoup(haystack.replace("\\","")[2:], 'html.parser')
@@ -108,7 +108,7 @@ def lambda_handler(event, context):
             # logger.info(len(posts))
             for idx, post in enumerate(posts):
                 wd_post_id = int(re.search('(?:<div class="post" id="post-)(\d*)', str(post)).group(1))
-                logger.info("Post " + str(idx) + ", ID " + str(wd_post_id))
+                # logger.info("Post " + str(idx) + ", ID " + str(wd_post_id))
                 subject = re.search('(?:<div class="title" id="post-title-\d*">\s*)([^\n]*)', str(post)).group(1)
                 # On a blank subject this returns as "</div>"
                 if subject == "</div>":
@@ -117,10 +117,10 @@ def lambda_handler(event, context):
                     username = re.search('(?:return false;">)([^<]*)(?:<\/a><\/span>,)', str(post)).group(1)
                     wd_user_id = int(re.search('(?:www\.wikidot\.com\/userkarma.php\?u=)([^\)]*)', str(post)).group(1))
                 except AttributeError: #NoneType, deleted user.
-                    logger.info('thread:')
-                    logger.info(wd_thread_id)
-                    logger.info('post:')
-                    logger.info(wd_post_id)
+                    # logger.info('thread:')
+                    # logger.info(wd_thread_id)
+                    # logger.info('post:')
+                    # logger.info(wd_post_id)
                     try:
                         wd_user_id = int(re.search('(?:data-id=")(\d*)', str(post)).group(1))
                         username = "Deleted Account " + str(wd_user_id)
@@ -135,7 +135,7 @@ def lambda_handler(event, context):
                                 wd_user_id = 0
                             except AttributeError: # This is getting ridiculous. More guest account types.
                                 try:
-                                    logger.info(str(post))
+                                    # logger.info(str(post))
                                     username = re.search('(?:&amp;default=http:\/\/www.wikidot.com/common--images/avatars/default/a16.png&amp;size=16"\/><\/a>)([^>]*)(?:<\/span>,)', str(post)).group(1)
                                     wd_user_id = 0
                                 except AttributeError:
@@ -188,7 +188,7 @@ def lambda_handler(event, context):
                 # logger.info('text is a ')
                 # logger.info(type(body))
             # While we could wait and send one big payload, that's a risky proposition on threads with lots of posts so let's not.
-            logger.info('out of the loop for a single page')
+            # logger.info('out of the loop for a single page')
             
             # Wrap the payload and send it, SCUTTLE can sort out posts it already has.
             outerpayload = {"wd_thread_id": int(wd_thread_id), "wd_forum_id": forum, 
@@ -215,7 +215,7 @@ def lambda_handler(event, context):
             output = json.dumps(outerpayload)
             headers = {"Authorization": "Bearer " + config.scuttle_token, "Content-Type": "application/json"}
             r = requests.put(callback_url + '/2stacks/thread/posts', data=output, headers=headers)
-            logger.info('Made a SCUTTLE Request!')
+            # logger.info('Made a SCUTTLE Request!')
             # logger.info('DATA: ')
             # logger.info(outerpayload)
 
diff --git a/2stacks-get-user-metadata.py b/2stacks-get-user-metadata.py
diff --git a/2stacks-list-all.pages.py b/2stacks-list-all.pages.py
diff --git a/2stacks-page-check-for-deletion b/2stacks-page-check-for-deletion
