Lookup scrapers by name, not by initial (#327)

fossasia · Dec 9, 2017 · 5ede95a · 5ede95a
1 parent 6db2da0
commit 5ede95a
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 17 deletions.
diff --git a/app/scrapers/exalead.py b/app/scrapers/exalead.py
@@ -20,6 +20,6 @@ def parseResponse(self, soup):
             urls.append({
                 'title': a.getText(),
                 'link': a.get('href')
-                })
+            })
         print('Exalead parsed: ' + str(urls))
         return urls
diff --git a/app/server.py b/app/server.py
@@ -1,5 +1,6 @@
-from flask import Flask, render_template, request, abort, Response, make_response
-from scrapers import feedgen
+from flask import (Flask, render_template, request, abort, Response,
+                   make_response)
+from scrapers import feedgen, scrapers
 from pymongo import MongoClient
 from dicttoxml import dicttoxml
 from xml.dom.minidom import parseString
@@ -46,9 +47,7 @@ def search(search_engine):
             abort(400, 'Not Found - undefined format')
 
         engine = search_engine
-        if engine not in ('google', 'bing', 'duckduckgo', 'yahoo', 'ask',
-                          'yandex', 'ubaidu', 'exalead', 'quora', 'tyoutube',
-                          'parsijoo', 'mojeek', 'vdailymotion'):
+        if engine not in scrapers:
             err = [404, 'Incorrect search engine', qformat]
             return bad_request(err)
 
@@ -69,7 +68,7 @@ def search(search_engine):
         for line in result:
             line['link'] = line['link'].encode('utf-8')
             line['title'] = line['title'].encode('utf-8')
-            if engine in ['b', 'a']:
+            if 'desc' in line:
                 line['desc'] = line['desc'].encode('utf-8')
 
         if qformat == 'json':
@@ -95,8 +94,5 @@ def set_header(r):
 
 
 if __name__ == '__main__':
-
-    app.run(
-        host='0.0.0.0',
-        port=int(os.environ.get('PORT', 7001)),
-        debug=args.dev)
+    port = int(os.environ.get('PORT', 7001))
+    app.run(host='0.0.0.0', port=port, debug=args.dev)
diff --git a/app/templates/index.html b/app/templates/index.html
@@ -61,13 +61,13 @@ <h1><code>query-server</code></h1>
                             <button type="submit" value="yahoo" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/yahoo_icon.ico') }}" width="30px" alt="Yahoo Icon"> Yahoo</button>
                             <button type="submit" value="ask" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/ask_icon.ico') }}" width="30px" alt="Ask Icon"> Ask</button>
                             <button type="submit" value="yandex" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/yandex_icon.png') }}" width="30px" alt="Yandex Icon"> Yandex</button>
-                            <button type="submit" value="ubaidu" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/baidu_icon.ico') }}" width="30px" alt="Baidu Icon"> Baidu</button>
+                            <button type="submit" value="baidu" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/baidu_icon.ico') }}" width="30px" alt="Baidu Icon"> Baidu</button>
                             <button type="submit" value="exalead" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/exalead_icon.png') }}" width="30px" alt="Exalead Icon"> Exalead</button>
                             <button type="submit" value="quora" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/quora_icon.png') }}" width="30px" alt="Quora Icon"> Quora</button>
                             <button type="submit" value="parsijoo" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/parsijoo_icon.png') }}" width="30px" alt="Parsijoo Icon"> Parsijoo</button>
                             <button type="submit" value="mojeek" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/mojeek_icon.png') }}" width="30px" alt="Mojeek Icon"> Mojeek</button>
-                            <button type="submit" value="tyoutube" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/youtube_icon.png') }}" width="30px" alt="YouTube Icon"> YouTube</button>
-                            <button type="submit" value="vdailymotion" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/dailymotion_icon.png') }}" width="30px" alt="Dailymotion Icon"> Dailymotion</button>
+                            <button type="submit" value="youtube" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/youtube_icon.png') }}" width="30px" alt="YouTube Icon"> YouTube</button>
+                            <button type="submit" value="dailymotion" class="btn btn-lg  search btn-outline"><img src="{{ url_for('static', filename='images/dailymotion_icon.png') }}" width="30px" alt="Dailymotion Icon"> Dailymotion</button>
                         </div>
                     </div>
                     <div class="col-sm-2">

diff --git a/app/test_server.py b/app/test_server.py
@@ -37,7 +37,7 @@ def make_engine_api_call(engine_name):
 
 @pytest.mark.xfail(PYTHON3 or not TRAVIS_CI, reason=REASON)
 def test_engine_api_calls(engine_names=None):
-    engines = ['ask', 'ubaidu', 'bing', 'duckduckgo', 'tyoutube',
-               'exalead', 'mojeek', 'google', 'quora', 'yahoo', 'yandex', 'parsijoo']
+    engines = ('ask', 'baidu', 'bing', 'dailymotion', 'duckduckgo', 'exalead',
+               'google', 'mojeek', 'parsijoo', 'quora', 'yahoo', 'yandex', 'youtube')
     for engine_name in (engine_names or engines):
         make_engine_api_call(engine_name)