Added unit tests for better code coverage (#368)

.travis.yml

@@ -11,8 +11,8 @@ install:
 before_script:
   - flake8 . --count --max-complexity=15 --show-source --statistics
 script:
-  - python app/server.py > /dev/null &
-  - pytest --cov=./ 
+  - python -m app.server > /dev/null &
+  - pytest --cov=./
   - kill $(lsof -t -i:7001)
 after_success:
   - bash <(curl -s https://codecov.io/bash)

app/server.py

@@ -1,12 +1,13 @@
 import json
 import os
 from argparse import ArgumentParser
+
 from defusedxml.minidom import parseString
 from dicttoxml import dicttoxml
 from flask import (Flask, Response, abort, jsonify, make_response,
                    render_template, request)
 
-from scrapers import feed_gen, scrapers
+from app.scrapers import feed_gen, scrapers
 
 DISABLE_CACHE = True  # Temporarily disable the MongoDB cache
 if DISABLE_CACHE:
@@ -27,11 +28,6 @@ def store(url, links):
     'error': 'Could not parse the page due to Internal Server Error'
 }
 
-parser = ArgumentParser()
-help_msg = "Start the server in development mode with debug=True"
-parser.add_argument("--dev", help=help_msg, action="store_true")
-args = parser.parse_args()
-
 
 @app.route('/')
 def index():
@@ -113,4 +109,8 @@ def set_header(r):
 
 if __name__ == '__main__':
     port = int(os.environ.get('PORT', 7001))
+    parser = ArgumentParser()
+    help_msg = "Start the server in development mode with debug=True"
+    parser.add_argument("--dev", help=help_msg, action="store_true")
+    args = parser.parse_args()
     app.run(host='0.0.0.0', port=port, debug=args.dev)

bandit.yml

@@ -0,0 +1 @@
+skips: ['B101']

requirements.txt

@@ -14,3 +14,4 @@ pytest-cov>=2.4.0
 requests>=2.13.0
 webencodings>=0.5
 defusedxml>=0.5.0
+mock>=2.0.0

test/test_ask.py

@@ -0,0 +1,58 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Ask
+
+
+def test_next_start():
+    assert 3 == Ask().next_start(2, None)
+
+
+def test_parse_response_for_none():
+    html_text = """<div class="PartialSearchResults-noresults">
+         <div class="PartialSearchResults-noresults-body">
+             <p>No results for:</p>
+             <p><b>44754546546545545465465f4654f654654</b></p>
+             <p>Please try again.</p>
+         </div>
+    </div>"""
+    stub_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = Ask().parse_response(stub_soup)
+    assert resp is None
+
+
+def test_parse_response_with_desc():
+    html_div = """<div class="PartialSearchResults-item" data-zen="true">
+        <div class="PartialSearchResults-item-title">
+            <a class="PartialSearchResults-item-title-link result-link"
+             href='mock_url'>mock_title</a>
+        </div>
+        <p class="PartialSearchResults-item-abstract">mock_desc</p>
+        </div>"""
+    stub_soup_div = BeautifulSoup(html_div, 'html.parser')
+    resp = Ask().parse_response(stub_soup_div)
+    expected_resp = [
+        {
+            'link': u'mock_url',
+            'title': u'mock_title',
+            'desc': u'mock_desc'
+        }
+    ]
+    assert resp == expected_resp
+
+
+def test_parse_response_without_desc():
+    html_div = """<div class="PartialSearchResults-item" data-zen="true">
+        <div class="PartialSearchResults-item-title">
+            <a class="PartialSearchResults-item-title-link result-link"
+             href='mock_url'>mock_title</a>
+        </div>
+        </div>"""
+    stub_soup_div = BeautifulSoup(html_div, 'html.parser')
+    resp = Ask().parse_response(stub_soup_div)
+    expected_resp = [
+        {
+            'link': u'mock_url',
+            'title': u'mock_title'
+        }
+    ]
+    assert resp == expected_resp

test/test_baidu.py

@@ -0,0 +1,16 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Baidu
+
+
+def test_parse_response():
+    html_text = """<div class="result c-container "><h3 class="t">
+        <a href="mock_url" target="_blank">mock_title</a>
+        </h3></div>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = Baidu().parse_response(dummy_soup)
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    assert resp == expected_resp

test/test_bing.py

@@ -0,0 +1,18 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Bing
+
+
+def test_parse_response():
+    html_text = """<li class="b_algo">
+        <h2><a href="mock_url">mock_title</h2>
+        <div class="b_caption"><p>mock_desc</p>
+        </div><li>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = Bing().parse_response(dummy_soup)
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url',
+        'desc': u'mock_desc'
+    }]
+    assert resp == expected_resp

test/test_duckduckgo.py

@@ -0,0 +1,16 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import DuckDuckGo
+
+
+def test_parse_response():
+    html_text = """<h2 class="result__title">
+        <a class="result__a" href="mock_url">mock_title</a>
+        </h2>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    resp = DuckDuckGo().parse_response(dummy_soup)
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    assert resp == expected_resp

test/test_generalized.py

@@ -0,0 +1,88 @@
+from mock import patch
+import pytest
+
+from app.scrapers.generalized import Scraper
+
+
+@patch('requests.models.Response')
+@patch('app.scrapers.generalized.requests.get')
+def test_get_page(mock_request_get, mock_response):
+    mock_request_get.return_value = mock_response
+    mock_response.url = "Mock Url"
+    response = Scraper().get_page("dummy_query")
+    assert response == mock_response
+    expected_payload = {'q': 'dummy_query', '': ''}
+    expected_headers = {
+        'User-Agent': (
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) '
+            'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 '
+            'Safari/537.36'
+        )
+    }
+    mock_request_get.assert_called_with(
+        '', headers=expected_headers, params=expected_payload)
+
+
+def test_parse_response():
+    with pytest.raises(NotImplementedError):
+        Scraper().parse_response(None)
+
+
+def test_next_start():
+    dummy_prev_results = ['dummy_value']
+    if not Scraper().next_start(3, dummy_prev_results) == 4:
+        raise AssertionError()
+
+
+@patch('app.scrapers.generalized.Scraper.parse_response')
+@patch('app.scrapers.generalized.Scraper.get_page')
+@patch('requests.models.Response')
+def test_search(mock_resp, mock_get_page, mock_parse_resp):
+    mock_get_page.return_value = mock_resp
+    mock_resp.text = "Mock response"
+    expected_resp = [{
+        'title': 'mock_title',
+        'link': 'mock_url'
+    }]
+    # assuming parse_response is being implemented by
+    # classes inheriting Scraper. Thus, returning dummy
+    # response instead of raising NotImplementedError
+    mock_parse_resp.return_value = expected_resp
+    resp = Scraper().search('dummy_query', 1)
+    assert resp == expected_resp
+
+
+@patch('app.scrapers.generalized.Scraper.get_page')
+@patch('requests.models.Response')
+def test_search_parsed_response_none(mock_resp, mock_get):
+    mock_get.return_value = mock_resp
+    mock_resp.text = "Mock Response"
+    with patch('app.scrapers.generalized.Scraper.parse_response',
+               return_value=None):
+        resp = Scraper().search('dummy_query', 1)
+        assert resp == []
+
+
+@patch('app.scrapers.generalized.requests.get')
+@patch('app.scrapers.generalized.Scraper.parse_response')
+@patch('requests.models.Response')
+def test_search_without_count(mock_resp, mock_parse_resp, mock_get):
+    mock_get.return_value = mock_resp
+    mock_resp.text = 'mock response'
+    expected_resp = [{
+        'title': 'mock_title',
+        'link': 'mock_url'
+    }]
+    expected_payload = {'q': 'dummy_query'}
+    expected_headers = {
+        'User-Agent': (
+            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) '
+            'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 '
+            'Safari/537.36'
+        )
+    }
+    mock_parse_resp.return_value = expected_resp
+    resp = Scraper().search_without_count('dummy_query')
+    assert resp == expected_resp
+    mock_get.assert_called_with(
+        '', headers=expected_headers, params=expected_payload)

test/test_google.py

@@ -0,0 +1,16 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Google
+
+
+def test_parse_response():
+    html_text = """<h3 class="r">
+    <a href="mock_url">mock_title</a>
+    </h3>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    resp = Google().parse_response(dummy_soup)
+    assert resp == expected_resp

test/test_mojeek.py

@@ -0,0 +1,14 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Mojeek
+
+
+def test_parse_response():
+    html_text = '<a href="mock_url" class="ob">mock_title</a>'
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url'
+    }]
+    resp = Mojeek().parse_response(dummy_soup)
+    assert resp == expected_resp

test/test_parsijoo.py

@@ -0,0 +1,22 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Parsijoo
+
+
+def test_parse_response():
+    html_text = """<div class="result">
+    <span class="result-title">
+    <a href="mock_url">""" + " " * 22 + """mock_title </a></span>
+    <span class="result-url">mock_url</span>
+    <span class="result-desc">""" + " " * 34 + """ mock_desc </span>
+    <span class="result-similar"><a href="mock_similar_link"
+    title="mock_similar_title">mock_similar</a>
+    </span></div>"""
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'mock_url',
+        'desc': u'mock_desc'
+    }]
+    resp = Parsijoo().parse_response(dummy_soup)
+    assert resp == expected_resp

test/test_quora.py

@@ -0,0 +1,16 @@
+from bs4 import BeautifulSoup
+
+from app.scrapers import Quora
+
+
+def test_parse_response():
+    html_text = "<div><a class='question_link' href='/mock_url'>" \
+        "<span class='question_text'><span class='rendered_qtext'>" \
+        "mock_title</span></span></a></div>"
+    dummy_soup = BeautifulSoup(html_text, 'html.parser')
+    expected_resp = [{
+        'title': u'mock_title',
+        'link': u'https://www.quora.com/mock_url'
+    }]
+    resp = Quora().parse_response(dummy_soup)
+    assert resp == expected_resp