diff --git a/.flake8.ini b/.flake8.ini index 9525ff8c..c83744b3 100644 --- a/.flake8.ini +++ b/.flake8.ini @@ -1,6 +1,4 @@ [flake8] -max-line-length = 150 -ignore = W191 exclude = .git, __pycache__ diff --git a/.travis.yml b/.travis.yml index 2389e898..d023e126 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,29 +1,15 @@ -matrix: - include: - - language: python - python: "2.7" - install: - - pip install flake8 pytest - - pip install -r requirements.txt - before_script: - - flake8 . --count --max-complexity=12 --max-line-length=85 --show-source --statistics - script: - - python app/server.py > /dev/null & - - pytest --capture=sys - - kill $(lsof -t -i:7001) - after_success: - - codecov - - - language: python - python: "3.6" - install: - - pip install flake8 pytest - - pip install -r requirements.txt - before_script: - - flake8 . --count --max-complexity=12 --max-line-length=85 --show-source --statistics - script: - - pytest --capture=sys - after_success: - - codecov - - +language: python +python: + - "2.7" + - "3.6" +install: + - pip install codecov flake8 pytest + - pip install -r requirements.txt +before_script: + - flake8 . --count --max-complexity=15 --max-line-length=85 --show-source --statistics +script: + - python app/server.py > /dev/null & + - pytest --capture=sys + - kill $(lsof -t -i:7001) +after_success: + - codecov diff --git a/app/scrapers/__init__.py b/app/scrapers/__init__.py index 90deb8f0..fc2b6047 100644 --- a/app/scrapers/__init__.py +++ b/app/scrapers/__init__.py @@ -1,18 +1,20 @@ from __future__ import print_function + import json import sys -from google import Google -from duckduckgo import Duckduckgo -from bing import Bing -from yahoo import Yahoo -from ask import Ask -from yandex import Yandex -from baidu import Baidu -from exalead import Exalead -from quora import Quora -from youtube import Youtube -from parsijoo import Parsijoo -from mojeek import Mojeek + +from .google import Google +from .duckduckgo import Duckduckgo +from .bing import Bing +from .yahoo import Yahoo +from .ask import Ask +from .yandex import Yandex +from .baidu import Baidu +from .exalead import Exalead +from .quora import Quora +from .youtube import Youtube +from .parsijoo import Parsijoo +from .mojeek import Mojeek scrapers = { 'g': Google(), diff --git a/app/scrapers/ask.py b/app/scrapers/ask.py index 42ba69f2..c7622bb2 100644 --- a/app/scrapers/ask.py +++ b/app/scrapers/ask.py @@ -1,6 +1,6 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Ask(Scraper): @@ -25,7 +25,7 @@ def parseResponse(self, soup): title = div.div.a.text url = div.div.a['href'] try: - p = div.find('p', {'class': 'PartialSearchResults-item-abstract'}) + p = div.find('p', class_='PartialSearchResults-item-abstract') desc = p.text.replace('\n', '') urls.append({'title': title, 'link': url, 'desc': desc}) except Exception: diff --git a/app/scrapers/baidu.py b/app/scrapers/baidu.py index 5011cc19..26636c4d 100644 --- a/app/scrapers/baidu.py +++ b/app/scrapers/baidu.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Baidu(Scraper): diff --git a/app/scrapers/bing.py b/app/scrapers/bing.py index 50be0c6c..ba9be2bf 100644 --- a/app/scrapers/bing.py +++ b/app/scrapers/bing.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Bing(Scraper): diff --git a/app/scrapers/duckduckgo.py b/app/scrapers/duckduckgo.py index a4879c32..7cd08931 100644 --- a/app/scrapers/duckduckgo.py +++ b/app/scrapers/duckduckgo.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Duckduckgo(Scraper): diff --git a/app/scrapers/exalead.py b/app/scrapers/exalead.py index 8ec14f18..ac5d8188 100644 --- a/app/scrapers/exalead.py +++ b/app/scrapers/exalead.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Exalead(Scraper): diff --git a/app/scrapers/google.py b/app/scrapers/google.py index fcaa5d22..87631f66 100644 --- a/app/scrapers/google.py +++ b/app/scrapers/google.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Google(Scraper): diff --git a/app/scrapers/mojeek.py b/app/scrapers/mojeek.py index 2caf7e4e..52e346ca 100644 --- a/app/scrapers/mojeek.py +++ b/app/scrapers/mojeek.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Mojeek(Scraper): diff --git a/app/scrapers/parsijoo.py b/app/scrapers/parsijoo.py index bf9c85e2..a70bad91 100644 --- a/app/scrapers/parsijoo.py +++ b/app/scrapers/parsijoo.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Parsijoo(Scraper): diff --git a/app/scrapers/quora.py b/app/scrapers/quora.py index 87967bc4..085a73dd 100644 --- a/app/scrapers/quora.py +++ b/app/scrapers/quora.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Quora(Scraper): diff --git a/app/scrapers/yahoo.py b/app/scrapers/yahoo.py index 65aa87c0..c9a11587 100644 --- a/app/scrapers/yahoo.py +++ b/app/scrapers/yahoo.py @@ -1,6 +1,10 @@ from __future__ import print_function -from generalized import Scraper -import urllib +from .generalized import Scraper + +try: + from urllib.parse import unquote # Python 3 +except ImportError: + from urllib import unquote # Python 2 class Yahoo(Scraper): @@ -15,7 +19,7 @@ def parseResponse(self, soup): """ Parse response and returns the urls Returns: urls (list) - [[Tile1,url1], [Title2, url2],..] + [[Tile1, url1], [Title2, url2], ...] """ urls = [] for h in soup.findAll('h3', attrs={'class': 'title'}): @@ -24,7 +28,7 @@ def parseResponse(self, soup): r = y.get('href') f = r.split('RU=') e = f[-1].split('/RK=1') - u = urllib.unquote(e[0]) + u = unquote(e[0]) urls.append({ 'title': y.getText(), 'link': u diff --git a/app/scrapers/yandex.py b/app/scrapers/yandex.py index c232af9d..05d14f69 100644 --- a/app/scrapers/yandex.py +++ b/app/scrapers/yandex.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Yandex(Scraper): diff --git a/app/scrapers/youtube.py b/app/scrapers/youtube.py index 0397637e..836cabd6 100644 --- a/app/scrapers/youtube.py +++ b/app/scrapers/youtube.py @@ -1,5 +1,5 @@ from __future__ import print_function -from generalized import Scraper +from .generalized import Scraper class Youtube(Scraper): diff --git a/app/server.py b/app/server.py index 63b7fe28..018ed53b 100644 --- a/app/server.py +++ b/app/server.py @@ -1,10 +1,13 @@ -from flask import Flask, render_template, request, abort, Response, make_response -from scrapers import feedgen -from pymongo import MongoClient -from dicttoxml import dicttoxml -from xml.dom.minidom import parseString import json import os +from xml.dom.minidom import parseString + +from dicttoxml import dicttoxml +from flask import (Flask, Response, abort, jsonify, make_response, + render_template, request) +from pymongo import MongoClient + +from scrapers import feedgen app = Flask(__name__) err = "" @@ -59,26 +62,24 @@ def search(search_engine): db['queries'].insert( {"query": query, "engine": engine, "qformat": qformat}) - for line in result: - line['link'] = line['link'].encode('utf-8') - line['title'] = line['title'].encode('utf-8') - if engine in ['b', 'a']: - line['desc'] = line['desc'].encode('utf-8') + try: + unicode # unicode is not defined in Python 3 + for line in result: + line['link'] = line['link'].encode('utf-8') + line['title'] = line['title'].encode('utf-8') + if 'desc' in line: + line['desc'] = line['desc'].encode('utf-8') + except NameError: + pass # Python 3 strings are already Unicode if qformat == 'json': - jsonfeed = json.dumps(result).encode('utf-8') - return Response(jsonfeed, mimetype='application/json') - xmlfeed = parseString( - (dicttoxml( - result, - custom_root='channel', - attr_type=False))).toprettyxml() + return jsonify(result) + xmlfeed = dicttoxml(result, custom_root='channel', attr_type=False) + xmlfeed = parseString(xmlfeed).toprettyxml() return Response(xmlfeed, mimetype='application/xml') - except Exception as e: print(e) - return Response(json.dumps(errorObj).encode( - 'utf-8'), mimetype='application/json') + return jsonify(errorObj) @app.after_request @@ -88,10 +89,4 @@ def set_header(r): if __name__ == '__main__': - app.run( - host='0.0.0.0', - port=int( - os.environ.get( - 'PORT', - 7001)), - debug=True) + app.run(host='0.0.0.0', port=int(os.environ.get('PORT', 7001)), debug=True) diff --git a/app/test_server.py b/app/test_server.py index b1bc612b..ac47222f 100644 --- a/app/test_server.py +++ b/app/test_server.py @@ -1,22 +1,10 @@ -""" -# Let's get pytest turned on before we start testing in earnest - -from .scrapers import small_test - - -def test_small_test(): - small_test() -""" - import os -import sys import pytest import requests -PYTHON3 = sys.version_info.major >= 3 -REASON = 'Python 3 blocked until #297 goes live' +REASON = 'Do you have query-server running on http://127.0.0.1:7001 ?' TRAVIS_CI = os.getenv('TRAVIS', False) # Running in Travis CI? @@ -24,7 +12,7 @@ def test_true(): assert True, "We have a problem!" -@pytest.mark.xfail(PYTHON3 or not TRAVIS_CI, reason=REASON) +@pytest.mark.xfail(not TRAVIS_CI, reason=REASON) def test_invalid_url_api_call(): response = requests.get('http://localhost:7001/api/v1/search/invalid_url') assert response.json()['Status Code'] == 404 @@ -35,9 +23,9 @@ def make_engine_api_call(engine_name): assert requests.get(url).json()['Status Code'] == 400, engine_name -@pytest.mark.xfail(PYTHON3 or not TRAVIS_CI, reason=REASON) +@pytest.mark.xfail(not TRAVIS_CI, reason=REASON) def test_engine_api_calls(engine_names=None): - engines = ['ask', 'ubaidu', 'bing', 'duckduckgo', 'tyoutube', - 'exalead', 'mojeek', 'google', 'quora', 'yahoo', 'yandex', 'parsijoo'] + engines = ('ask', 'ubaidu', 'bing', 'duckduckgo', 'tyoutube', + 'exalead', 'mojeek', 'google', 'quora', 'yahoo', 'yandex') for engine_name in (engine_names or engines): make_engine_api_call(engine_name)