Skip to content

Commit

Permalink
AR bills: remove all premature yield statements
Browse files Browse the repository at this point in the history
  • Loading branch information
chrisyamas committed Apr 26, 2023
1 parent d5337ab commit 38e0204
Showing 1 changed file with 12 additions and 6 deletions.
18 changes: 12 additions & 6 deletions scrapers/ar/bills.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,19 @@


class ARBillScraper(Scraper):
"""
The challenge here is that I need to comment out all
"""

ftp_user = ""
ftp_pass = ""
bills = {}

def scrape(self, chamber=None, session=None):

self.ftp_user = os.environ.get("AR_FTP_USER", None)
self.ftp_pass = os.environ.get("AR_FTP_PASSWORD", None)
self.ftp_user = os.environ.get("AR_FTP_USER", "arklegview")
self.ftp_pass = os.environ.get("AR_FTP_PASSWORD", "V1ew0nly!")

if not self.ftp_user or not self.ftp_pass:
self.error("AR_FTP_USER and AR_FTP_PASSWORD env variables are required.")
Expand All @@ -38,10 +43,13 @@ def scrape(self, chamber=None, session=None):
leg_chambers = [chamber] if chamber else ["upper", "lower"]

for leg_chamber in leg_chambers:
yield from self.scrape_bill(leg_chamber, session)
self.scrape_bill(leg_chamber, session)

self.scrape_actions()

if not self.bills:
raise EmptyScrape

for bill_id, bill in self.bills.items():
yield bill

Expand Down Expand Up @@ -107,7 +115,7 @@ def scrape_bill(self, chamber, session):
version_url = f"https://www.arkleg.state.ar.us/assembly/{year}/{self.slug}/Bills/{bill_url}.pdf"
bill.add_version_link(bill_id, version_url, media_type="application/pdf")

yield from self.scrape_bill_page(bill)
self.scrape_bill_page(bill)

self.bills[bill_id] = bill

Expand Down Expand Up @@ -326,8 +334,6 @@ def scrape_bill_page(self, bill):
media_type="application/pdf",
)

yield bill

# TODO: reincorporate below code block for vote processing
# (commented out due to scraper having vote event processing issue)
# for link in page.xpath(
Expand Down

0 comments on commit 38e0204

Please sign in to comment.