first commit

crawler.py

@@ -0,0 +1,34 @@
+from page_source import PageSource
+from page_parser import PageParser
+from entry import Entry
+
+class Crawler:
+
+    spawns = 0
+    __keyword = None
+    __parser = None
+    __questions = []
+    __answers = []
+    __url = []
+    __entries = []
+
+    def __init__(self, search):
+        self.__parser = PageParser(keyword=search)
+        Crawler.spawns = Crawler.spawns + 1
+
+    def get_entries(self):
+        return self.__entries
+
+    def crawl(self):
+        self.__questions = self.__parser.get_all_questions()
+        self.__answers = self.__parser.get_all_answers()
+        self.__url = self.__parser.get_all_url()
+        self.__pack()
+
+    def __pack(self):
+        for i in range(0, len(self.__questions)):
+            try:
+                self.__entries.append(Entry(question=self.__questions[i], answer=self.__answers[i], url=self.__url[i]))
+            except IndexError:
+                pass
+

csv_manager.py

@@ -0,0 +1,21 @@
+import csv
+
+
+class CSVManager:
+
+    __filename = ""
+    __entries = set()
+
+    def __init__(self, filename, entries):
+        self.__filename = filename + ".csv"
+        self.__entries = entries
+
+    def save(self):
+        header = ['question', 'answer', 'url']
+        with open(file=self.__filename, mode='w', encoding='UTF-8') as file:
+            writer = csv.writer(file)
+            writer.writerow(header)
+            for entry in self.__entries:
+                list = [entry.question, entry.answer, entry.url]
+                writer.writerow(list)
+

entry.py

@@ -0,0 +1,21 @@
+class Entry:
+
+    question = None
+    answer = None
+    url = None
+
+    def __init__(self, question, answer, url):
+        self.question = question
+        self.answer = answer
+        self.url = url
+
+    def __eq__(self, other):
+        if isinstance(other, Entry):
+            if hash(self.question) == hash(other.question):
+                return True
+            else:
+                return False
+        return False
+
+    def __hash__(self):
+        return hash(self.question)

main.py

@@ -0,0 +1,13 @@
+from query_manager import QueryManager
+from csv_manager import CSVManager
+
+if __name__ == '__main__':
+    keyword = input("Search: ")
+    print("Please wait... Retrieving results...")
+    print("This might take a minute or two...")
+    print("\n")
+    manager = QueryManager(keyword=keyword)
+    manager.start()
+    print("Found Results: " + str(manager.current_index))
+    CSVManager(keyword, manager.get_entries()).save()
+

page_parser.py

@@ -0,0 +1,43 @@
+import time
+
+from bs4 import BeautifulSoup
+from page_source import PageSource
+
+class PageParser:
+
+    __html = None
+
+    def __init__(self, keyword):
+        text = PageSource(search_keyword=keyword)
+        time.sleep(1)
+        self.__html = BeautifulSoup(str(text), 'html5lib')
+
+
+    def get_all_questions(self):
+        list = []
+        container = self.__html.find_all('span', attrs={"class": "C7GS5b rkGIWe"})
+        for entry in container:
+            list.append(entry.find_next("div", attrs={"class": "BNeawe s3v9rd AP7Wnd"}).text)
+        return list
+
+    def get_all_answers(self):
+        list = []
+        container = self.__html.find_all('span', attrs={"class": "C7GS5b rkGIWe"})
+        for entry in container:
+            try:
+                answer = entry.find_next_sibling("div", attrs={"class": "kCrYT"}).find_next("div", attrs={"class": "BNeawe s3v9rd AP7Wnd"}).find_next("div", attrs={"class": "BNeawe s3v9rd AP7Wnd"})
+                list.append(answer.text)
+            except AttributeError:
+                pass
+        return list
+
+    def get_all_url(self):
+        list = []
+        container = self.__html.find_all("div", attrs={"class": "BNeawe UPmit AP7Wnd UwRFLe"})
+        for entry in container:
+            try:
+                answer = entry.text
+                list.append(answer)
+            except AttributeError:
+                pass
+        return list

page_source.py

@@ -0,0 +1,36 @@
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+
+class PageSource:
+
+    __driver = None
+    __keyword = None
+
+    def __init__(self, search_keyword):
+        options = webdriver.ChromeOptions()
+        # options.add_experimental_option("detach", True)
+        options.add_argument('--disable-javascript')
+        options.add_argument("--headless=new")
+        prefs = {}
+        prefs["webkit.webprefs.javascript_enabled"] = False
+        prefs["profile.content_settings.exceptions.javascript.*.setting"] = 2
+        prefs["profile.default_content_setting_values.javascript"] = 2
+        prefs["profile.managed_default_content_settings.javascript"] = 2
+        options.add_experimental_option("prefs", prefs)
+        self.__driver = webdriver.Chrome(options=options);
+        self.__driver.get("https://www.google.com")
+        self.__keyword = search_keyword
+        self.__get_search_field().send_keys(self.__keyword)
+        self.__get_search_button().submit()
+
+    def __get_html(self):
+        return str(self.__driver.page_source)
+
+    def __get_search_field(self):
+        return self.__driver.find_element(by=By.NAME, value="q")
+
+    def __get_search_button(self):
+        return self.__driver.find_element(by=By.NAME, value="btnK")
+
+    def __str__(self):
+        return str(self.__get_html())

python.csv

@@ -0,0 +1,73 @@
+question,answer,url
+What are the four classifications of system design?,"Four specific types of engineered system context are generally recognized in systems engineering : product system , service system , enterprise system and system of systems .",sebokwiki.org › wiki › Types_of_Systems
+What is the design of a real-time system?,Real-time systems are therefore usually designed as cooperating processes with a real-time executive controlling these processes. Collect information from sensors. May buffer information collected in response to a sensor stimulus. Carries out processing of collected information and computes the system response.,ifs.host.cs.st-andrews.ac.uk › Books › Presentations › PDF
+What is system design and its purpose in management information system?,"Definition: Systems design is the process of defining elements of a system like modules, architecture, components and their interfaces and data for a system based on the specified requirements.",www.toptal.com › designers › design-systems › benefits-of-design-system
+What are 5 examples of real-time software?,"A Real Time Operating System, commonly known as an RTOS, is a software component that rapidly switches between tasks, giving the impression that multiple programs are being executed at the same time on a single processing core.",www.geeksforgeeks.org › real-time-systems
+What is a system design meaning?,"System design is the process of designing the elements of a system such as the architecture, modules, and components, the different interfaces of those components, and the data that goes through that system.",swimm.io › learn › system-design-complete-guide-with-patterns-examples-...
+What are the criteria for system design?,"Performance, modifiability, availability, scalability, reliability, etc. are important quality requirements in system design. These 'ilities' are what we need to analyze for a system and determine if our system is designed properly.",www.linkedin.com › pulse › 5-key-considerations-while-designing-system-...
+What is the meaning and importance of system design?,"To Meet Specific Requirements: System design is necessary to ensure that a system meets specific requirements, such as performance, reliability, scalability, and maintainability. Without a clear design plan, a system may not function as intended, leading to performance issues, downtime, or other problems.",www.sciencedirect.com › topics › engineering › design-system
+Can Python be used for realtime applications?,Yes! You can certainly write a real-time application in Python.,medium.datadriveninvestor.com › why-should-you-choose-python-for-real...
+What is the use of Python in realtime applications?,"One of the prime uses of Python is that it is used by software developers. Python simplifies the software development process for complex apps. It is used for project management, as a support programming language, to build control, and testing.",www.mygreatlearning.com › Blog › AI and Machine Learning
+What is system design in MIS?,"What is Systems Design. Definition: Systems design is the process of defining elements of a system like modules, architecture, components and their interfaces and data for a system based on the specified requirements.",www.geeksforgeeks.org › why-is-it-important-to-learn-system-design
+What are the major design issues in a real-time system?,"Though all the Major Design Issues will be applicable to most of the Operating Systems. The important issues related to Operating system are transparency, flexibility, reliability, performance, scalability, naming, replication, synchronization, security.",www.mhhe.com › compsci › pressman › information › olc › RTdesign
+What is the design phase of a system?,"In the design phase, one or more designs are created to achieve the project result. Depending on the project subject, the design phase products include dioramas, flow-charts, sketches, site trees, HTML screen designs, photo impressions, prototypes, and UML schemas.",idapgroup.com › blog › sdlc-design-phase
+What are the 5 criteria for system design?,"In successful systems design, three main components must be considered and managed effectively. These are quality, timeliness and cost-effectiveness.",info.knowledgeleader.com › the-3-components-of-a-successful-systems-des...
+What is system development design?,"Introduction: Systems development is the process of defining, designing, testing, and implementing a new software application or program. It could include the internal development of customized systems, the creation of database systems, or the acquisition of third party developed software.",www.openxcell.com › Blog
+What are real-time systems in computer?,"Real-time computing (RTC) is the computer science term for hardware and software systems subject to a ""real-time constraint"", for example from event to system response. Real-time programs must guarantee response within specified time constraints, often referred to as ""deadlines"".",en.wikipedia.org › wiki › Real-time_computing
+What is called a real time operating system?,"A Real Time Operating System, commonly known as an RTOS, is a software component that rapidly switches between tasks, giving the impression that multiple programs are being executed at the same time on a single processing core.",www.highintegritysystems.com › rtos › what-is-an-rtos
+What are the 3 major components of the system design phase?,"In successful systems design, three main components must be considered and managed effectively. These are quality, timeliness and cost-effectiveness.",info.knowledgeleader.com › the-3-components-of-a-successful-systems-des...
+Is Python good for real time applications?,"Compatible with Real-Time Software
+
+ Python is easy to use and for this reason, it finds its way into the real-time embedded system as the recommended programming language. The software codes are in Python and not C or C++.",medium.datadriveninvestor.com › why-should-you-choose-python-for-real...
+What is system design phase?,"System design is the phase that bridges the gap between problem domain and the existing system in a manageable way. This phase focuses on the solution domain, i.e. “how to implement?” It is the phase where the SRS document is converted into a format that can be implemented and decides how the system will operate.",segwitz.com › what-is-system-design-and-why-it-is-necessary
+Why is system design important in MIS?,"To Meet Specific Requirements: System design is necessary to ensure that a system meets specific requirements, such as performance, reliability, scalability, and maintainability. Without a clear design plan, a system may not function as intended, leading to performance issues, downtime, or other problems.",www.indeed.com › Career Guide › Career development
+What is the most important factor in design of real-time systems?,"Determinism: Real-time systems must exhibit deterministic behavior, which means that the system's behavior must be predictable and repeatable for a given input. The system must always produce the same output for a given input, regardless of the load or other factors.",ifs.host.cs.st-andrews.ac.uk › Books › Presentations › PDF
+What is the structure of a real-time system?,"A real-time system takes inputs through sensors, and other external input devices, then processes these input signals in real-time to produce output. Therefore, a real-time system is implemented with the help of a variety of hardware components and software applications.",www.geeksforgeeks.org › characteristics-of-real-time-systems
+What can you actually use Python for?,"Compatible with Real-Time Software
+
+ Python is easy to use and for this reason, it finds its way into the real-time embedded system as the recommended programming language. The software codes are in Python and not C or C++.",www.datacamp.com › About Python › Learn Python
+What is the importance of design phase in system development life cycle?,"The purpose of the Design Phase is to transform the requirements into complete and detailed system design specifications. Once the design is approved, the Development Team begins the Development Phase.",www.fca.gov › template-fca › download › ITManual › itsystemsdevelopment
+How Python is used in real life?,"One of Python's key benefits is its ability to automate manual, repetitive tasks. With Python, you can learn how to automate just about anything by using either built-in modules or pre-written code from its robust library. Or you can write your own custom scripts to perform specific actions.",github.blog › 2023-03-02-why-python-keeps-growing-explained
+What is system design and issues?,"Definition: Systems design is the process of defining elements of a system like modules, architecture, components and their interfaces and data for a system based on the specified requirements.",economictimes.indiatimes.com › definition › systems-design
+Which of the following are real time systems?,"Data streaming, radar systems, customer service systems, and bank ATMs are all examples of real-time processing, as they require instantaneous processing to function efficiently.",testbook.com › question-answer › real-time-systems-are-_______--6284d9...
+What is the purpose of system design phase?,The purpose of the System Design process is to provide sufficient detailed data and information about the system and its system elements to enable the implementation consistent with architectural entities as defined in models and views of the system architecture.,doit.maryland.gov › SDLC Phase 05 Design Phase Single Custom
+What is the design phase of system development?,"Design Phase
+
+ These include: Identifying potential risks and defining mitigating design features. Performing a security risk assessment. Developing a conversion plan to migrate current data to the new system • Determining the operating environment.",info.knowledgeleader.com › the-3-components-of-a-successful-systems-des...
+What are types of system design?,"Therefore, a systematic approach is needed to manage the system requirements and design methodology. It can be classified as logical design and physical design. The logical design represents the abstract data flow, while the physical design represents the system's input and output processes.",www.educba.com › ... › Software Development Basics
+Which is the example of real time operating system?,"RTOS Examples
+
+ VxWorks: VxWorks is a real time operating system developed by Wind River Systems. It is widely used in the aerospace, defense, and industrial automation industries. QNX: QNX is a commercial real time operating system developed by BlackBerry.",www.prepbytes.com › blog › what-is-a-real-time-operating-system-rtos
+Why Python is mostly used in real world applications?,"Since it functions on cross-platform operating systems, Python can be used to develop a host of applications, including web apps, gaming apps, enterprise-level applications, ML apps, image processing, text processing, and so much more.",www.quora.com › Is-Python-good-for-real-time-applications
+What are real-time systems used for?,A real-time operating system (RTOS) is a special-purpose operating system used in computers that has strict time constraints for any job to be performed. It is employed mostly in those systems in which the results of the computations are used to influence a process while it is executing.,www.javatpoint.com › real-time-operating-system
+What is system design in management information system?,"System design involves the process in which an organization, in an appropriate situation, develops a newer system or strategy to complement or replace an existing one. This design and development cycle includes planning, analysis, design, implementation and maintenance.",www.sciencedirect.com › topics › engineering › design-system
+Why is system design important in information system?,"System design helps software engineers identify potential problems that can affect the performance of the system. By understanding the system's architecture and its components, engineers can identify and fix problems before they become too costly to fix. This can help improve the overall performance of the system.",www.linkedin.com › pulse › what-concept-system-design-how-its-importan...
+What are examples of real-time computing systems?,"Typical examples of real-time systems include Air Traffic Control Systems, Networked Multimedia Systems, Command Control Systems etc.",users.ece.cmu.edu › ~koopman › des_s99 › real_time
+What is the importance of system design phase in software development?,"System design helps software engineers identify potential problems that can affect the performance of the system. By understanding the system's architecture and its components, engineers can identify and fix problems before they become too costly to fix. This can help improve the overall performance of the system.",www.tutorialspoint.com › system_analysis_and_design › system_design
+What is the reason for system design?,"A design system helps product teams define and standardize visual elements like color, typography, spacing, and imagery. These elements express the heart and soul of your brand and form a visual language. A well-defined visual language can enhance the perceived quality of your product.",www.geeksforgeeks.org › why-is-it-important-to-learn-system-design
+What are the advantages of system design in MIS?,"System design enables developers to identify and eliminate unnecessary components and processes, create software systems quickly and easily, and improve the security, scalability, user experience, and maintenance of the software system.",www.cerritos.edu › dwhitney › SitePages › CIS201 › Lectures
+What is system design and its purpose?,The goal of system design is to allocate the requirements of a large system to hardware and software components. The system design activity starts after the system requirements analysis has been completed.,www.tutorialspoint.com › system_analysis_and_design › system_analysis_a...
+What is Python with used for?,"Python is commonly used for developing websites and software, task automation, data analysis, and data visualization. Since it's relatively easy to learn, Python has been adopted by many non-programmers such as accountants and scientists, for a variety of everyday tasks, like organizing finances.",www.coursera.org › Coursera Articles › Data
+What is a realtime applications Python?,"A real-time application, or RTA, is an application that functions within a time frame that the user senses as immediate or current.",www.techtarget.com › searchunifiedcommunications › real-time-applicatio...
+What are the components of system design?,"Each component in a design system meets a specific interaction or UI need, and has been created to work together to provide intuitive user experiences. An avatar, badge, dropdown menu, icon, logo, page layout, spinner, and tag are all examples of components.",www.geeksforgeeks.org › what-are-the-components-of-system-design
+Which programming language is best for real-time applications?,"A real-time application, or RTA, is an application that functions within a time frame that the user senses as immediate or current.",www.upgrad.com › Blog › Data Science
+What is a real-time system with example?,"A real-time system means that the system is subjected to real-time, i.e., the response should be guaranteed within a specified timing constraint or the system should meet the specified deadline. For example flight control systems, real-time monitors, etc.",www.sciencedirect.com › topics › computer-science › real-time-systems
+How is Python used in our daily life?,"Python is extensively applied in data science, data analysis, machine learning, data engineering, web development, software development, and other fields. To learn more about what Python is, you can find our blog article on the fundamentals of this popular language.",www.mygreatlearning.com › Blog › AI and Machine Learning
+What are the two types of system design?,"1) Architectural design: To describes the views, models, behaviour, and structure of the system. 2) Logical design: To represent the data flow, inputs and outputs of the system.",economictimes.indiatimes.com › Definition › Software-Development
+What are the applications of real time processing?,"Some real-world applications of real-time processing are found in banking systems, data streaming, customer service structures, and weather radars.",www.hpe.com › what-is › real-time-processing
+What are the major components of the system design phase?,"System design is the process of defining the architecture, components, modules, interfaces, and data for a computer system. It involves analyzing the requirements of the system, identifying the constraints and assumptions, and defining the high-level structure and components of the system.",info.knowledgeleader.com › the-3-components-of-a-successful-systems-des...
+How many phases are there in system designing?,"These included planning, creating, developing, testing, and deploying. Note that it left out the major stages of analysis and maintenance. What Are the 7 Phases of SDLC? The new seven phases of SDLC include planning, analysis, design, development, testing, implementation, and maintenance.",www.nasa.gov › reference › 4-0-system-design-processes
+What are 5 examples of real-time operating system?,"Real-time computing (RTC) is the computer science term for hardware and software systems subject to a ""real-time constraint"", for example from event to system response. Real-time programs must guarantee response within specified time constraints, often referred to as ""deadlines"".",users.ece.cmu.edu › ~koopman › des_s99 › real_time
+What are the phases of system design?,"There are usually six stages in this cycle: requirement analysis, design, development and testing, implementation, documentation, and evaluation.",blog.hubspot.com › website › design-system
+What Python means?,"Python is an interpreted, object-oriented, high-level programming language with dynamic semantics developed by Guido van Rossum. It was originally released in 1991. Designed to be easy as well as fun, the name ""Python"" is a nod to the British comedy group Monty Python.",www.teradata.com › Glossary › What-is-Python
+What are real-time applications?,"Real-Time Applications are applications that operate within an immediate time frame; sensing, analyzing, and acting on streaming data as it happens. This is in contrast to a database-centric application where information is ingested and stored in a database (in the cloud or on-premise) for future analysis.",vantiq.com › what-are-real-time-applications
+What are examples of real-time applications?,"Some real-world applications of real-time processing are found in banking systems, data streaming, customer service structures, and weather radars.",vantiq.com › what-are-real-time-applications
+What are design issues of real time operating system?,"Are the processing components powerful enough? A CPU with really high utilization will lead to unpredictable Realtime behavior. Also, it is possible that the high priority tasks in the system will starve the low priority tasks of any CPU time. This can cause the low priority tasks to misbehave.",www.tutorialspoint.com › basic-model-of-a-real-time-system
+What are two applications of real-time system?,"Examples of real-time systems include digital control, command and control, signal processing, and telecommunication systems. DIGITAL CONTROL: They are the simplest and the most deterministic real-time applications. Many real-time systems are embedded in sensors and actuators and function as digital controllers.",aits-tpt.edu.in › wp-content › uploads › 2018/08 › RTS-Unit-1
+What are the following are real-time systems?,"Data streaming, radar systems, customer service systems, and bank ATMs are all examples of real-time processing, as they require instantaneous processing to function efficiently.",testbook.com › Operating Systems
+What are the design issues of operating system?,"Though all the Major Design Issues will be applicable to most of the Operating Systems. The important issues related to Operating system are transparency, flexibility, reliability, performance, scalability, naming, replication, synchronization, security.",www.ques10.com › discuss-operating-system-design-issues-1
+What are the 4 types of system design?,"There are four system design processes: developing stakeholder expectations, technical requirements, logical decompositions, and design solutions. Figure 4.0-1 illustrates the recursive relationship among the four system design processes.",www.linkedin.com › pulse › what-concept-system-design-how-its-importan...
+What are the five activities included in systems design?,"These six activities are: design the environment, design application architecture and software, design user interfaces, design system interfaces, design the database, and design system controls and security.",towardsdatascience.com › system-design-101-b8f15162ef7c
+How many types of system design are there?,"Therefore, a systematic approach is needed to manage the system requirements and design methodology. It can be classified as logical design and physical design. The logical design represents the abstract data flow, while the physical design represents the system's input and output processes.",www.nasa.gov › reference › 4-0-system-design-processes
+What are the 5 phases in the system development process?,"SDLC is not a methodology per se, but rather a description of the phases that a methodology should address. The list of phases is not definitive, but typically includes planning, analysis, design, build, test, implement, and maintenance/support.",www.clouddefense.ai › blog › system-development-life-cycle
+What is the main purpose of system design?,The goal of system design is to allocate the requirements of a large system to hardware and software components.,segwitz.com › what-is-system-design-and-why-it-is-necessary

query_manager.py

@@ -0,0 +1,30 @@
+import threading
+from crawler import Crawler
+
+class QueryManager:
+
+    question_list = []
+    entry_set = set()
+    current_index = 0;
+    keyword = ""
+
+    def __init__(self, keyword):
+        QueryManager.question_list.append(keyword)
+
+    def start(self):
+        for i in range(20):
+            x = threading.Thread(target=self.__spawn_crawler())
+            x.start()
+            x.join()
+
+    def get_entries(self):
+        return QueryManager.entry_set
+
+    def __spawn_crawler(self):
+        crawler = Crawler(QueryManager.question_list[-1])
+        crawler.crawl()
+        for entry in crawler.get_entries():
+            QueryManager.question_list.append(entry.question)
+            QueryManager.entry_set.add(entry)
+        QueryManager.current_index = len(QueryManager.entry_set)
+