Fdu FdZddlmZddlZddlmZddlmZddlm Z m Z ddl m Z ddl mZdd lmZdd lmZdd lmZdd lmZdd lmZdd lmZdd lmZdd l mZ!dd l"mZ#dd l$mZ%dd l&mZdd l'mZ(dd l)mZ*dd l+m,Z-ddl.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z6ddl7m8Z8ddl9m:Z:ddl;mZ>m?Z?ddl@mAZAmBZBddlCmDZDeeze!ze(zZEeeFjGjGZHe:dddddddddddeDd3d&ZId4d(ZJd5d,ZKd6d.ZLd6d/ZM d7d8d2ZNdS)9zSelenium web scraping module.) annotationsN)Path)platform)OptionalType) BeautifulSoup)WebDriverException)Options)Service) WebDriver)By)expected_conditions) WebDriverWait)ChromeDriverManager)GeckoDriverManager)EdgeChromiumDriverManager)Agent)command)logger) MemoryItem get_memory)extract_hyperlinksformat_hyperlinks) validate_urlbrowse_websitezBrowses a WebsitestringzThe URL to visitT)type descriptionrequiredz$What you want to find on the website)urlquestionr strr!agentrreturncp t||\}}n<#t$r/}|jdd}d|cYd}~Sd}~wwxYwt |t |||||}t ||}t|dkr |dd}t|d|d|S)aBrowse a website and return the answer and links to the user Args: url (str): The url of the website to browse question (str): The question asked by the user Returns: Tuple[str, WebDriver]: The answer and links to the user and the webdriver  rzError: NzAnswer gathered from website: z Links: ) scrape_text_with_seleniumr msgsplit add_headersummarize_memorize_webpagescrape_links_with_seleniumlen close_browser) r r!r#drivertexter)summarylinkss QC:\Users\Administrator\Downloads\Auto-GPT-master\autogpt\commands\web_selenium.pyrr*s.0e<<  ekk$"  v(dHeVLLG &vs 3 3E 5zzA~~bqb & GG G G G GGs A$A A Atuple[WebDriver, str]ctjdtjtt t td}||jj }| d|jj dkrf|jj rd|_ | dttt|}nv|jj dkr>t#t%t'|}n(|jj d krt)| }nt*d ks t*d kr*| d | d| d|jj r*| d| dt-d}t/|rt3t5|n,t3t7|}||t;|dt?j tBj"df|#d}tI|d}|ddgD]}|%|&} d| 'D} d| D} d(d| D} || fS)zScrape text from a website using selenium Args: url (str): The url of the website to scrape Returns: Tuple[WebDriver, str]: The webdriver and the text scraped from the website selenium)chromeedgefirefoxsafariz~user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36r;Tz --disable-gpu)serviceoptionsr:r<)r>linuxlinux2z--disable-dev-shm-usagez--remote-debugging-port=9222z --no-sandboxz--headless=newz/usr/bin/chromedriver bodyzreturn document.body.outerHTML; html.parserscriptstylec3>K|]}|VdSN)strip).0lines r5 z,scrape_text_with_selenium..s* 8 8dTZZ\\ 8 8 8 8 8 8c3nK|]0}|dD]}|V1dS)z N)r*rH)rIrJphrases r5rKz,scrape_text_with_selenium..sC N NTZZ=M=M N N6fllnn N N N N N N NrLr&c3K|]}||V dSrG)rIchunks r5rKz,scrape_text_with_selenium..s'88u%8U888888rL))logging getLoggersetLevelCRITICAL ChromeOptions EdgeOptionsFirefoxOptions SafariOptionsconfigselenium_web_browser add_argumentselenium_headlessheadless FirefoxDriverGeckoDriverServicerinstall EdgeDriverEdgeDriverServiceEdgeDriverManager SafariDriverrr ChromeDriverexistsChromeDriverServicer"rgetruntilECpresence_of_element_locatedr TAG_NAMEexecute_scriptrextractget_text splitlinesjoin) r r#options_availabler>r0chromium_driver_path page_sourcesouprDr1lineschunkss r5r(r(TsX j!!**7+;<<< ! ::S/ 0QRTTG  I |(I55 < ) 2#G    1 1 1&'9';';'C'C'E'EFFPW     *f 4 4%&7&9&9&A&A&C&CDDg     *h 6 6g... w  (h"6"6  !: ; ; ;  !? @ @ @^,,, < ) 2  !1 2 2 2   1 1 1#$;<<#**,,F',@(A(ABBB$%8%:%:%B%B%D%DEE      JJsOOO&"## & V'<== ''(IJJK m 4 4D$'*++ ==??D 8 8doo&7&7 8 8 8E N N N N NF 9988888 8 8D 4<rLr0r list[str]c|j}t|d}|ddgD]}|t||}t |S)zScrape links from a website using selenium Args: driver (WebDriver): The webdriver to use to scrape the links Returns: List[str]: The links scraped from the website rCrDrE)rurrorr)r0r rurvrD hyperlinkss r5r-r-si$K m 4 4D$'*++#D#..J Z ( ((rLNonec.|dS)zoClose the browser Args: driver (WebDriver): The webdriver to close Returns: None N)quit)r0s r5r/r/s KKMMMMMrLc ttdd5}|}dddn #1swxYwY||dS#t$r}t d|Yd}~dSd}~wwxYw)zAdd a header to the website Args: driver (WebDriver): The webdriver to use to add the header Returns: None z/js/overlay.jsrNzError executing overlay.js: )openFILE_DIRreadrn Exceptionprint)r0 overlay_fileoverlay_scriptr2s r5r+r+s2 X---s 3 3 1|)..00N 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1n----- 222 0Q001111111112s4A; A?A?A B'A??Br1Optional[WebDriver]c|sdSt|}tjd|dt|j}t j|||j|}|||jS)a1Summarize text using the OpenAI API Args: url (str): The url of the text text (str): The text to summarize question (str): The question to ask the model driver (WebDriver): The webdriver to use to scroll the page Returns: str: The summary of the text zError: No text to summarizez Text length: z characters)r!) r.rinforrZr from_webpageaddr3)r r1r!r#r0 text_lengthmemory new_memorys r5r,r,s~$ -,,d))K K8 888999  % %F(sEL8TTTJ JJz  rL)r r"r!r"r#rr$r")r r"r#rr$r6)r0r r r"r$ry)r0r r$r|rG) r r"r1r"r!r"r#rr0rr$r")O__doc__ __future__rrRpathlibrsysrtypingrrbs4rselenium.common.exceptionsr !selenium.webdriver.chrome.optionsr rV!selenium.webdriver.chrome.servicer rh#selenium.webdriver.chrome.webdriverr rfselenium.webdriver.common.byr selenium.webdriver.edge.optionsrWselenium.webdriver.edge.servicerc!selenium.webdriver.edge.webdriverrb"selenium.webdriver.firefox.optionsrX"selenium.webdriver.firefox.servicer`$selenium.webdriver.firefox.webdriverr_#selenium.webdriver.remote.webdriver!selenium.webdriver.safari.optionsrY#selenium.webdriver.safari.webdriverreselenium.webdriver.supportrrkselenium.webdriver.support.waitrwebdriver_manager.chromerwebdriver_manager.firefoxrwebdriver_manager.microsoftrrdautogpt.agent.agentrautogpt.command_decoratorr autogpt.logsrautogpt.memory.vectorrrautogpt.processing.htmlrrautogpt.url_utils.validatorsrBrowserOptions__file__parentrrr(r-r/r+r,rPrLr5rsW##""""""!!!!!!!!999999FFFFFFLLLLLLIIIIII++++++BBBBBBHHHHHHEEEEEEHHHHHHLLLLLLKKKKKK999999FFFFFFIIIIII@@@@@@999999888888888888VVVVVV%%%%%%------88888888IIIIIIII555555,~= M 4>> '  1CQUVVA    HHH  H:IIIIX))))(    2222,#' rL