Added unit tests for better code coverage (#368)

.travis.yml

@@ -11,8 +11,8 @@ install:
 before_script:
   - flake8 . --count --max-complexity=15 --show-source --statistics
 script:
-  - python app/server.py > /dev/null &
-  - pytest --cov=./ 
+  - python -m app.server > /dev/null &
+  - pytest --cov=./
   - kill $(lsof -t -i:7001)
 after_success:
   - bash <(curl -s https://codecov.io/bash)

app/server.py

@@ -1,12 +1,13 @@
 import json
 import os
 from argparse import ArgumentParser
+
 from defusedxml.minidom import parseString
 from dicttoxml import dicttoxml
 from flask import (Flask, Response, abort, jsonify, make_response,
                    render_template, request)
 
-from scrapers import feed_gen, scrapers
+from app.scrapers import feed_gen, scrapers
 
 DISABLE_CACHE = True  # Temporarily disable the MongoDB cache
 if DISABLE_CACHE:
@@ -27,11 +28,6 @@ def store(url, links):
     'error': 'Could not parse the page due to Internal Server Error'
 }
 
-parser = ArgumentParser()
-help_msg = "Start the server in development mode with debug=True"
-parser.add_argument("--dev", help=help_msg, action="store_true")
-args = parser.parse_args()
-
 
 @app.route('/')
 def index():
@@ -112,4 +108,8 @@ def set_header(r):
 
 if __name__ == '__main__':
     port = int(os.environ.get('PORT', 7001))
+    parser = ArgumentParser()
+    help_msg = "Start the server in development mode with debug=True"
+    parser.add_argument("--dev", help=help_msg, action="store_true")
+    args = parser.parse_args()
     app.run(host='0.0.0.0', port=port, debug=args.dev)

requirements.txt

@@ -14,3 +14,4 @@ pytest-cov>=2.4.0
 requests>=2.13.0
 webencodings>=0.5
 defusedxml>=0.5.0
+mock>=2.0.0

test/test_ask.py

@@ -0,0 +1,62 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Ask
+
+
+def test_next_start():
+    if 3 != Ask().next_start(2, None):
+        raise AssertionError()
+
+
+def test_parse_response_for_none():
+    html_text = """<div class="PartialSearchResults-noresults">
+         <div class="PartialSearchResults-noresults-body">
+             <p>No results for:</p>
+             <p><b>44754546546545545465465f4654f654654</b></p>
+             <p>Please try again.</p>
+         </div>
+    </div>"""
+    stub_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = Ask().parse_response(stub_soup)
+    if resp:
+        raise AssertionError()
+
+
+def test_parse_response_with_desc():
+    html_div = """<div class="PartialSearchResults-item" data-zen="true">
+        <div class="PartialSearchResults-item-title">
+            <a class="PartialSearchResults-item-title-link result-link"
+             href='mock_url'>mock_title</a>
+        </div>
+        <p class="PartialSearchResults-item-abstract">mock_desc</p>
+        </div>"""
+    stub_soup_div = BeautifulSoup(html_div, 'html.parser')
+    resp = Ask().parse_response(stub_soup_div)
+    expected_resp = [
+        {
+            'link': u'mock_url',
+            'title': u'mock_title',
+            'desc': u'mock_desc'
+        }
+    ]
+    if not resp == expected_resp:
+        raise AssertionError()
+
+
+def test_parse_response_without_desc():
+    html_div = """<div class="PartialSearchResults-item" data-zen="true">
+        <div class="PartialSearchResults-item-title">
+            <a class="PartialSearchResults-item-title-link result-link"
+             href='mock_url'>mock_title</a>
+        </div>
+        </div>"""
+    stub_soup_div = BeautifulSoup(html_div, 'html.parser')
+    resp = Ask().parse_response(stub_soup_div)
+    expected_resp = [
+        {
+            'link': u'mock_url',
+            'title': u'mock_title'
+        }
+    ]
+    if not resp == expected_resp:
+        raise AssertionError()

test/test_baidu.py

@@ -0,0 +1,17 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Baidu
+
+
+def test_parse_response():
+    html_text = """<div class="result c-container "><h3 class="t">
+        <a href="mock_url" target="_blank">mock_title</a>
+        </h3></div>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = Baidu().parse_response(dummy_soup)
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    if not resp == expected_resp:
+        raise AssertionError()

test/test_bing.py

@@ -0,0 +1,19 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Bing
+
+
+def test_parse_response():
+    html_text = """<li class="b_algo">
+        <h2><a href="mock_url">mock_title</h2>
+        <div class="b_caption"><p>mock_desc</p>
+        </div><li>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = Bing().parse_response(dummy_soup)
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url',
+        'desc': u'mock_desc'
+    }]
+    if not expected_resp == resp:
+        raise AssertionError()

test/test_duckduckgo.py

@@ -0,0 +1,17 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import DuckDuckGo
+
+
+def test_parse_response():
+    html_text = """<h2 class="result__title">
+        <a class="result__a" href="mock_url">mock_title</a>
+        </h2>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = DuckDuckGo().parse_response(dummy_soup)
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    if not resp == expected_resp:
+        raise AssertionError()

test/test_google.py

@@ -0,0 +1,17 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Google
+
+
+def test_parse_response():
+    html_text = """<h3 class="r">
+    <a href="mock_url">mock_title</a>
+    </h3>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    resp = Google().parse_response(dummy_soup)
+    if not resp == expected_resp:
+        raise AssertionError()

test/test_mojeek.py

@@ -0,0 +1,15 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Mojeek
+
+
+def test_parse_response():
+    html_text = '<a href="mock_url" class="ob">mock_title</a>'
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    resp = Mojeek().parse_response(dummy_soup)
+    if not resp == expected_resp:
+        raise AssertionError()

test/test_parsijoo.py

@@ -0,0 +1,23 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Parsijoo
+
+
+def test_parse_response():
+    html_text = """<div class="result">
+    <span class="result-title">
+    <a href="mock_url">""" + " " * 22 + """mock_title </a></span>
+    <span class="result-url">mock_url</span>
+    <span class="result-desc">""" + " " * 34 + """ mock_desc </span>
+    <span class="result-similar"><a href="mock_similar_link"
+    title="mock_similar_title">mock_similar</a>
+    </span></div>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url',
+        'desc': u'mock_desc'
+    }]
+    resp = Parsijoo().parse_response(dummy_soup)
+    if not resp == expected_resp:
+        raise AssertionError()

test/test_quora.py

@@ -0,0 +1,17 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Quora
+
+
+def test_parse_response():
+    html_text = "<div><a class='question_link' href='/mock_url'>" \
+        "<span class='question_text'><span class='rendered_qtext'>" \
+        "mock_title</span></span></a></div>"
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'https://www.quora.com/mock_url'
+    }]
+    resp = Quora().parse_response(dummy_soup)
+    if not resp == expected_resp:
+        raise AssertionError()