eswyft/acmespb-py
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixes, improvements. added text filter.

Browse Source
This commit is contained in:
Evgeny Zinoviev 2021-02-16 12:54:42 +03:00
parent f01e27a00b
commit 94463e4926
13 changed files with 322 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -5,10 +5,6 @@ __pycache__/
venv/
instance/
.pytest_cache/
.coverage
htmlcov/
dist/
build/
*.egg-info/

13
app/acmespb.py → acmespb.py
View File

@ -102,10 +102,13 @@ def offers(query, target_url=None, page=1):
soup = BeautifulSoup(r.text, "html.parser")
p = soup.find("p", class_="red")
if p:
total_matches = int(re.findall("([0-9]+)", p.string)[0])
pages = math.ceil(total_matches / per_page)
try:
total_matches = int(re.findall("([0-9]+)", p.string)[0])
pages = math.ceil(total_matches / per_page)
except IndexError:
raise AcmeException(p.string)
offers = []
offer_list = []
for trow in soup.find_all('div', class_='trow'):
if 'thead' in trow['class']:
continue
@ -128,6 +131,6 @@ def offers(query, target_url=None, page=1):
acmepharm = AcmePharmacy(name=phname, address=address, phone=phone, geo=geo)
acmeoffer = AcmeOffer(name=name, country=country, price=price, pharmacy=acmepharm)
offers.append(acmeoffer)
offer_list.append(acmeoffer)
return target_url, pages, offers
return target_url, pages, offer_list

68
app.py Normal file
View File

@ -0,0 +1,68 @@
import logging
import traceback
import acmespb
from flask import Flask, render_template, jsonify, request
app = Flask(__name__)
app.config.from_mapping(SECRET_KEY='dev', JSON_AS_ASCII=False)
logger = logging.getLogger('app')
@app.after_request
def add_header(r):
"""
Add headers to both force latest IE rendering engine or Chrome Frame,
and also to cache the rendered page for 10 minutes.
"""
r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
r.headers["Pragma"] = "no-cache"
r.headers["Expires"] = "0"
r.headers['Cache-Control'] = 'public, max-age=0'
return r
@app.route('/', methods=['GET'])
def index():
return render_template('index.html')
@app.route('/hints.ajax', methods=['GET'])
def ajax_hints():
query = request.args.get('q') or ''
if len(query) < 3:
return jsonify(error="query is too short")
results = acmespb.search(query)
return jsonify(response=results)
@app.route('/offers.ajax', methods=['GET'])
def ajax_offers():
query = request.args.get('q') or ''
page = request.args.get('page') or 1
target_url = request.args.get('target_url') or ''
try:
if page == 1 or not target_url:
target_url, trade_names = acmespb.trade_names(query)
if trade_names:
return jsonify(tradeNames=trade_names)
target_url, pages, offers = acmespb.offers(query, page=page, target_url=target_url)
except Exception as e:
traceback.print_exc()
return jsonify(error=str(e))
return jsonify(
offers=[offer.as_dict() for offer in offers],
pages=pages
)
# TODO support empty results
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000)

91
app/__init__.py
View File

@ -1,91 +0,0 @@
import os
import time
from . import acmespb
from flask import Flask, render_template
from flask_socketio import SocketIO, emit
socketio = SocketIO()
def create_app(test_config=None):
app = Flask(__name__, instance_relative_config=True)
app.config.from_mapping(
SECRET_KEY='dev',
DATABASE=os.path.join(app.instance_path, 'app.sqlite'),
)
if test_config is None:
# load the instance config, if it exists, when not testing
app.config.from_pyfile('config.py', silent=True)
else:
# load the test config if passed in
app.config.from_mapping(test_config)
# ensure the instance folder exists
try:
os.makedirs(app.instance_path)
except OSError:
pass
socketio.init_app(app)
@app.route('/')
def hello():
return render_template('index.html')
@socketio.on('get_hints')
def handle_get_hints_event(q):
print('[get_hints] id=%d, query=%s' % (q['id'], q['query']))
if len(q['query']) < 3:
response = {
'id': q['id'],
'error': "query is too short"
}
emit('hints', response)
return
results = acmespb.search(q['query'])
response = {
'id': q['id'],
'response': results
}
emit('hints', response)
@socketio.on('get_offers')
def handle_get_offers_event(q):
print('[get_offers] id=%d, query=%s' % (q['id'], q['query']))
target_url, trade_names = acmespb.trade_names(q['query'])
if trade_names:
response = {
'id': q['id'],
"response": trade_names
}
emit('hints', response)
return
page = 1
pages = 0
target_url = None
while pages == 0 or page <= pages:
target_url, pages, offers = acmespb.offers(q['query'], page=page, target_url=target_url)
print("[%d] pages=%d, target_url=%s" % (page, pages, target_url))
response = {
'id': q['id'],
'offers': [offer.as_dict() for offer in offers],
'page': page,
'pages': pages
}
emit('offers', response)
time.sleep(0.5)
page += 1
response = {
'id': q['id'],
'end': True
}
emit('offers', response)
# TODO empty response
return app

191
app/static/app.js
View File

@ -1,191 +0,0 @@
class Search {
constructor() {
this.searchDebounced = _.debounce((query) => {
if (query.length < 3)
return;
this.socket.emit('get_hints', {
id: this.updateRequestId(),
query
});
}, 150);
let field = document.getElementById('queryInput');
let btn = document.getElementById('querySubmit');
this.autoComplete = new Autocomplete(field, {
data: [],
maximumItems: 10,
onInput: (value) => {
this.searchDebounced(value);
},
onSelectItem: ({label}) => {
// console.log('selected:', label)
},
highlightClass: 'text-danger'
});
btn.addEventListener('click', this.onSubmit);
field.addEventListener('keydown', this.onInputKeyDown);
this.btn = btn;
this.field = field;
this.socket = io();
this.socket.on('hints', this.onHints);
this.socket.on('offers', this.onOffers)
}
updateRequestId() {
this.requestId = requestId();
return this.requestId;
}
onInputKeyDown = (e) => {
if (e.keyCode === 10 || e.keyCode === 13)
this.onSubmit();
}
onSubmit = (e) => {
if (this.isLocked())
return;
this.lockButton('Загрузка...');
gMaps.removeAllPoints();
this.socket.emit('get_offers', {
id: this.updateRequestId(),
query: this.field.value
});
}
onHints = (data) => {
if (data.id !== this.requestId)
return;
this.unlockButton();
if (data.error) {
console.warn(data.error);
return;
}
this.autoComplete.setData(data.response.map(item => {
return {label: item, value: ''};
}));
this.autoComplete.renderIfNeeded();
}
onOffers = (data) => {
if (data.id !== this.requestId)
return;
if (data.end) {
this.unlockButton();
return;
} else {
this.lockButton(data.pages > 1 ? `${data.page} из ${data.pages}` : null);
}
for (let offer of data.offers)
gMaps.addOffer(offer);
}
isLocked() {
return this.btn.classList.contains('disabled');
}
lockButton(text) {
if (text !== null)
this.btn.innerText = text;
this.btn.classList.add('disabled');
}
unlockButton() {
this.btn.classList.remove('disabled');
this.btn.innerText = 'Поиск';
}
}
class Maps {
constructor() {
/**
* @type {ymaps.Map}
*/
this.map = null;
ymaps.ready(this.onInit);
this.places = {};
}
onInit = () => {
this.map = new ymaps.Map("mapContainer", {
center: [59.94, 30.32],
zoom: 11
});
this.map.controls.remove('searchControl');
}
addPoint({geo, offersRef, hint, pharmacyName, pharmacyAddress, pharmacyPhone}) {
let mark = new ymaps.Placemark(geo, {
hintContent: hint,
}, {
preset: 'islands#dotIcon',
openEmptyBalloon: true,
iconColor: '#3caa3c'
});
mark.events.add('balloonopen', e => {
let lines = offersRef.map(offer => {
return `${offer.name} (${offer.price} руб.)`
});
let html = `<b>${pharmacyName}</b><br>`;
html += `${pharmacyAddress}<br>`;
html += `тел: ${pharmacyPhone}<br><br>`;
html += lines.join('<br>');
mark.properties.set('balloonContent', html);
});
this.map.geoObjects.add(mark);
return mark;
}
removeAllPoints() {
this.map.geoObjects.removeAll();
}
addOffer(offer) {
// console.log('[addOffer]', offer);
let hash = offer.pharmacy.hash;
if (hash in this.places)
this.places[hash].offers.push(offer);
else {
this.places[hash] = {
offers: [offer],
};
this.places[hash].mark = this.addPoint({
geo: offer.pharmacy.geo,
hint: offer.pharmacy.name,
pharmacyName: offer.pharmacy.name,
pharmacyAddress: offer.pharmacy.address,
pharmacyPhone: offer.pharmacy.phone,
offersRef: this.places[hash].offers
});
}
}
}
function requestId() {
return _.random(1, 99999999);
}
let gMaps, gSearch;
window.addEventListener('DOMContentLoaded', function() {
gSearch = new Search();
gMaps = new Maps();
// document.getElementById('test').addEventListener('click', () => {
// gMaps.addTestPoint();
// });
});

2
app/templates/index.html
View File

@ -1,2 +0,0 @@
{% extends "base.html" %}

2
requirements.txt
View File

@ -1,6 +1,4 @@
requests~=2.25.1
eventlet
requests[socks]
beautifulsoup4~=4.9.3
Flask~=1.1.2
Flask-SocketIO

7
server.py
View File

@ -1,7 +0,0 @@
#!/bin/env python
from app import create_app, socketio
app = create_app()
if __name__ == '__main__':
socketio.run(app, host='0.0.0.0')

235
static/app.js Normal file
View File

@ -0,0 +1,235 @@
class Search {
constructor() {
this.searchDebounced = _.debounce((query) => {
if (query.length < 3)
return;
fetch(`/hints.ajax?q=${encodeURIComponent(query)}`)
.then(response => {
if (!response.ok)
throw new Error(`statusText is ${response.statusText}`);
return response.json();
})
.then(({response, error}) => {
this.unlockButton();
if (error)
throw new Error(error);
this.autoComplete.setData(response.map(item => {
return {label: item, value: ''};
}));
this.autoComplete.renderIfNeeded();
})
}, 150);
this._filterDebounced = _.debounce((e) => {
let filter = e.target;
globalMaps.setFilter(filter.value);
}, 500)
let field = document.getElementById('queryInput');
let btn = document.getElementById('querySubmit');
let filterField = document.getElementById('filterInput');
this.autoComplete = new Autocomplete(field, {
data: [],
maximumItems: 10,
onInput: (value) => {
this.searchDebounced(value);
},
onSelectItem: ({label}) => {
// console.log('selected:', label)
},
highlightClass: 'text-danger'
});
btn.addEventListener('click', this.onSubmit);
field.addEventListener('keydown', this.onInputKeyDown);
filterField.addEventListener('input', this._filterDebounced);
this.btn = btn;
this.field = field;
}
onInputKeyDown = (e) => {
if (e.keyCode === 10 || e.keyCode === 13)
this.onSubmit();
}
getOffers(query, page) {
fetch(`/offers.ajax?q=${encodeURIComponent(this.field.value)}&page=${page}`)
.then(response => {
if (!response.ok)
throw new Error(`statusText is ${response.statusText}`);
return response.json();
})
.then(({error, tradeNames, end, offers, pages}) => {
if (error)
throw new Error(error);
if (tradeNames) {
this.autoComplete.setData(tradeNames.map(item => {
return {label: item, value: ''};
}));
this.autoComplete.renderIfNeeded();
return this.unlockButton();
}
for (let offer of offers)
globalMaps.addOffer(offer);
if (page >= pages) {
return this.unlockButton();
} else {
this.lockButton(pages > 1 ? `${page} из ${pages}` : null);
setTimeout(() => {
this.getOffers(query, page + 1);
}, 1000)
}
})
.catch((error) => {
console.error(error);
alert(error);
this.unlockButton();
})
}
onSubmit = (e) => {
if (this.isLocked())
return;
this.lockButton('Загрузка...');
globalMaps.removeAllPoints();
this.getOffers(this.field.value, 1);
}
isLocked() {
return this.btn.classList.contains('disabled');
}
lockButton(text) {
if (text !== null)
this.btn.innerText = text;
this.btn.classList.add('disabled');
}
unlockButton() {
this.btn.classList.remove('disabled');
this.btn.innerText = 'Поиск';
}
}
class Maps {
constructor() {
/**
* @type {ymaps.Map}
*/
this.map = null;
ymaps.ready(this.onInit);
this.filter = null;
this.places = {};
}
onInit = () => {
this.map = new ymaps.Map("mapContainer", {
center: [59.94, 30.32],
zoom: 11
});
this.map.controls.remove('searchControl');
}
addPoint({geo, offersRef, hint, pharmacyName, pharmacyAddress, pharmacyPhone}) {
let mark = new ymaps.Placemark(geo, {
hintContent: hint,
}, {
preset: 'islands#dotIcon',
openEmptyBalloon: true,
iconColor: '#3caa3c'
});
mark.events.add('balloonopen', e => {
let lines = offersRef.map(offer => {
return `${offer.name} (${offer.price} руб.)`
});
let html = `<b>${pharmacyName}</b><br>`;
html += `${pharmacyAddress}<br>`;
html += `тел: ${pharmacyPhone}<br><br>`;
html += lines.join('<br>');
mark.properties.set('balloonContent', html);
});
this.map.geoObjects.add(mark);
return mark;
}
removeAllPoints() {
this.map.geoObjects.removeAll();
}
addOffer(offer) {
// console.log('[addOffer]', offer);
let hash = offer.pharmacy.hash;
if (hash in this.places)
this.places[hash].offers.push(offer);
else
this.places[hash] = {offers: [offer]};
if (!this.places[hash].mark && this.isAllowed(offer.name))
this.showPlaceOnMap(hash, offer);
}
showPlaceOnMap(hash, offer) {
this.places[hash].mark = this.addPoint({
geo: offer.pharmacy.geo,
hint: offer.pharmacy.name,
pharmacyName: offer.pharmacy.name,
pharmacyAddress: offer.pharmacy.address,
pharmacyPhone: offer.pharmacy.phone,
offersRef: this.places[hash].offers
});
}
hidePlaceFromMap(hash) {
if (this.places[hash].mark) {
this.map.geoObjects.remove(this.places[hash].mark);
delete this.places[hash].mark;
}
}
setFilter(filter) {
if (!filter)
filter = null;
this.filter = filter;
for (let hash in this.places) {
if (!this.places.hasOwnProperty(hash))
continue;
let pl = this.places[hash];
let ok = pl.offers.filter(o => this.isAllowed(o.name))
if (pl.mark && !ok.length)
this.hidePlaceFromMap(hash);
else if (!pl.mark && ok.length)
this.showPlaceOnMap(hash, ok[0]);
}
}
isAllowed(productName) {
return this.filter === null || productName.indexOf(this.filter) !== -1;
}
}
let globalMaps = null;
let globalSearch = null;
window.addEventListener('DOMContentLoaded', function() {
globalSearch = new Search();
globalMaps = new Maps();
});

2
app/static/autocomplete.js → static/autocomplete.js
View File

@ -96,7 +96,7 @@ class Autocomplete {
this.field.value = e.target.innerText;
if (this.options.onSelectItem)
this.options.onSelectItem({
value: e.target.value,
value: e.target.dataset.value,
label: e.target.innerText,
});
this.dropdown.hide();

0
app/static/style.css → static/style.css
View File

9
app/templates/base.html → templates/index.html
View File

@ -12,10 +12,10 @@
<script src="https://api-maps.yandex.ru/2.1/?apikey=ce936229-3ef4-41b1-96c0-270bcf8ff341&lang=ru_RU" type="text/javascript"></script>
<script src="{{ url_for('static', filename='autocomplete.js') }}"></script>
<script src="{{ url_for('static', filename='app.js') }}"></script>
<script src="{{ url_for('static', filename='autocomplete.js') }}?1"></script>
<script src="{{ url_for('static', filename='app.js') }}?4"></script>
<title>{% block title %}{% endblock %}</title>
<title>ACMESPB здорового человека</title>
</head>
<body class="h-100 mh-100">
<div class="container h-100 pt-4 pb-4">
@ -23,7 +23,8 @@
<div>
<form class="mb-4" onsubmit="return false">
<div class="input-group">
<input type="text" class="form-control" id="queryInput" placeholder="Введите название препарата" autocomplete="off">
<input type="text" class="col-sm-7 form-control" id="queryInput" placeholder="Введите название препарата" autocomplete="off" style="flex: 2 1 auto;">
<input type="text" class="form-control" id="filterInput" placeholder="Фильтр" autocomplete="off">
<button type="submit" class="btn btn-outline-primary" id="querySubmit">Поиск</button>
</div>
</form>

8
app/test.py → test.py
View File

@ -1,14 +1,16 @@
import acmespb
import sys
from pprint import pprint
if __name__ == "__main__":
q_empty = "Волекам"
q_many = "Верошпирон"
#pprint(acmespb.trade_names("Марена красильная корневища и корни"))
page = 1
pages = 0
target_url = None
while pages == 0 or page <= pages:
target_url, pages, offers = acmespb.offers("Верошпирон", page=page, target_url=target_url)
target_url, pages, offers = acmespb.offers(q_empty, page=page, target_url=target_url)
print("[%d] pages=%d, target_url=%s" % (page, pages, target_url))
for offer in offers:
print(offer.as_dict())