abderr
/
ambar2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Version 2.1.18

Browse Source
master v2.1.18
Ilya.P 8 years ago
parent eae54a9750
commit a143182063
28 changed files with 94 additions and 135 deletions
Show Stats Download Patch File Download Diff File

10
FrontEnd/src/components/Search/components/SearchInput/SearchInput.js
Unescape Escape View File

@ -7,8 +7,6 @@ import MediaQuery from 'react-responsive'
import classes from './SearchInput.scss' import classes from './SearchInput.scss'
class SearchInput extends Component { class SearchInput extends Component {
static timeoutId = null
componentDidMount() { componentDidMount() {
this.refs.search_input.focus() this.refs.search_input.focus()
} }
@ -50,16 +48,8 @@ class SearchInput extends Component {
return return
} }
}} }}
onKeyDown={(event) => {
clearTimeout(this.timeoutId)
}}
onChange={(event, newValue) => { onChange={(event, newValue) => {
clearTimeout(this.timeoutId)
setQuery(newValue) setQuery(newValue)
this.timeoutId = setTimeout(() => {
search(0, newValue)
}, 500)
}} }}
underlineShow={false} underlineShow={false}
/> />

9
FrontEnd/src/index.html
Unescape Escape View File

@ -1,5 +1,6 @@
<!doctype html> <!doctype html>
<html lang="en"> <html lang="en">
<head> <head>
<title>Loading...</title> <title>Loading...</title>
<meta charset="utf-8"> <meta charset="utf-8">
@ -12,14 +13,10 @@
<meta name="apple-mobile-web-app-title" content="Ambar"> <meta name="apple-mobile-web-app-title" content="Ambar">
<meta name="application-name" content="Ambar"> <meta name="application-name" content="Ambar">
<meta name="theme-color" content="#ffffff"> <meta name="theme-color" content="#ffffff">
<!-- start Mixpanel -->
<script type="text/javascript">(function(e,a){if(!a.__SV){var b=window;try{var c,l,i,j=b.location,g=j.hash;c=function(a,b){return(l=a.match(RegExp(b+"=([^&]*)")))?l[1]:null};g&&c(g,"state")&&(i=JSON.parse(decodeURIComponent(c(g,"state"))),"mpeditor"===i.action&&(b.sessionStorage.setItem("_mpcehash",g),history.replaceState(i.desiredHash||"",e.title,j.pathname+j.search)))}catch(m){}var k,h;window.mixpanel=a;a._i=[];a.init=function(b,c,f){function e(b,a){var c=a.split(".");2==c.length&&(b=b[c[0]],a=c[1]);b[a]=function(){b.push([a].concat(Array.prototype.slice.call(arguments,
0)))}}var d=a;"undefined"!==typeof f?d=a[f]=[]:f="mixpanel";d.people=d.people||[];d.toString=function(b){var a="mixpanel";"mixpanel"!==f&&(a+="."+f);b||(a+=" (stub)");return a};d.people.toString=function(){return d.toString(1)+".people (stub)"};k="disable time_event track track_pageview track_links track_forms register register_once alias unregister identify name_tag set_config reset people.set people.set_once people.increment people.append people.union people.track_charge people.clear_charges people.delete_user".split(" ");
for(h=0;h<k.length;h++)e(d,k[h]);a._i.push([b,c,f])};a.__SV=1.2;b=e.createElement("script");b.type="text/javascript";b.async=!0;b.src="undefined"!==typeof MIXPANEL_CUSTOM_LIB_URL?MIXPANEL_CUSTOM_LIB_URL:"file:"===e.location.protocol&&"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js".match(/^\/\//)?"https://cdn.mxpnl.com/libs/mixpanel-2-latest.min.js":"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js";c=e.getElementsByTagName("script")[0];c.parentNode.insertBefore(b,c)}})(document,window.mixpanel||[]);
</script>
<!-- end Mixpanel -->
</head> </head>
<body> <body>
<div id="root" style="height: 100%"></div> <div id="root" style="height: 100%"></div>
</body> </body>
</html> </html>

21
FrontEnd/src/routes/CoreLayout/modules/CoreLayout.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { titles, stateValueExtractor, urls, constants, analytics } from 'utils' import { titles, stateValueExtractor, urls, constants } from 'utils'
import 'whatwg-fetch' import 'whatwg-fetch'
const CHANGE_FIELD = 'CORE.CHANGE_FIELD' const CHANGE_FIELD = 'CORE.CHANGE_FIELD'
@ -24,9 +24,6 @@ export const loadConfig = () => {
dispatch(changeField('version', apiInfo.version)) dispatch(changeField('version', apiInfo.version))
dispatch(changeField('lang', apiInfo.uiLang)) dispatch(changeField('lang', apiInfo.uiLang))
analytics(apiInfo.analyticsToken)
analytics().register({ apiUrl: urls.apiHost })
}) })
.then(() => dispatch(stopLoadingIndicator())) .then(() => dispatch(stopLoadingIndicator()))
.catch(error => { .catch(error => {
@ -56,7 +53,10 @@ const stopLoadingIndicator = () => {
const getApiUrl = () => new Promise((resolve, reject) => { const getApiUrl = () => new Promise((resolve, reject) => {
fetch('apiUrl.txt', { fetch('apiUrl.txt', {
method: 'GET' method: 'GET',
mode: 'cors',
credentials: 'include',
cache: 'no-cache'
}) })
.then(resp => resolve(resp.text())) .then(resp => resolve(resp.text()))
.catch(err => reject(err)) .catch(err => reject(err))
@ -64,7 +64,10 @@ const getApiUrl = () => new Promise((resolve, reject) => {
const getLocalizationsJson = () => new Promise((resolve, reject) => { const getLocalizationsJson = () => new Promise((resolve, reject) => {
fetch('localizations.json', { fetch('localizations.json', {
method: 'GET' method: 'GET',
mode: 'cors',
credentials: 'include',
cache: 'no-cache'
}) })
.then(resp => resolve(resp.text())) .then(resp => resolve(resp.text()))
.catch(err => reject(err)) .catch(err => reject(err))
@ -72,7 +75,10 @@ const getLocalizationsJson = () => new Promise((resolve, reject) => {
const getWebApiInfo = (url) => new Promise((resolve, reject) => { const getWebApiInfo = (url) => new Promise((resolve, reject) => {
fetch(url, { fetch(url, {
method: 'GET' method: 'GET',
mode: 'cors',
credentials: 'include',
cache: 'no-cache'
}) })
.then(resp => resolve(resp.json())) .then(resp => resolve(resp.json()))
.catch(err => reject(err)) .catch(err => reject(err))
@ -89,7 +95,6 @@ export function handleError(error, showErrorMessage = false) {
if (error.constructor === Response) { if (error.constructor === Response) {
error = `Response ${error.status} ${error.statusText} at ${error.url}` error = `Response ${error.status} ${error.statusText} at ${error.url}`
} }
analytics().event('ERROR', { description: error ? error.toString() : 'no info' })
console.log(error) console.log(error)
return { return {

3
FrontEnd/src/routes/MainLayout/modules/MainLayout.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { urls, titles, stateValueExtractor, analytics } from 'utils' import { urls, titles, stateValueExtractor } from 'utils'
import { push } from 'react-router-redux' import { push } from 'react-router-redux'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout' import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
@ -44,7 +44,6 @@ export function stopLoadingIndicator() {
export const toggleRateUsModal = (value) => { export const toggleRateUsModal = (value) => {
return (dispatch, getState) => { return (dispatch, getState) => {
dispatch(changeField('showRateUsModal', value)) dispatch(changeField('showRateUsModal', value))
analytics().event('ACCOUNT.RATE_US_MODAL_OPENED')
} }
} }

2
FrontEnd/src/routes/SearchPage/modules/DetailedView.js
Unescape Escape View File

@ -1,6 +1,5 @@
import { stateValueExtractor } from 'utils/' import { stateValueExtractor } from 'utils/'
import { hitsModel } from 'models/' import { hitsModel } from 'models/'
import { analytics } from 'utils'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout' import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout' import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -25,7 +24,6 @@ export const loadHighlight = (fileId, query) => {
.then((resp) => { .then((resp) => {
dispatch(setContentHighlight(fileId, hitsModel.contentHighlightFromApi(resp))) dispatch(setContentHighlight(fileId, hitsModel.contentHighlightFromApi(resp)))
dispatch(startStopHighlightLoadingIndicator(fileId, false)) dispatch(startStopHighlightLoadingIndicator(fileId, false))
analytics().event('SEARCH.LOAD_HIGHLIGHT')
}) })
.catch((errorPayload) => { .catch((errorPayload) => {
dispatch(startStopHighlightLoadingIndicator(fileId, false)) dispatch(startStopHighlightLoadingIndicator(fileId, false))

4
FrontEnd/src/routes/SearchPage/modules/FileVisibilityReducer.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { stateValueExtractor, analytics } from 'utils' import { stateValueExtractor } from 'utils'
import { hitsModel } from 'models/' import { hitsModel } from 'models/'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout' import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout' import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -18,7 +18,6 @@ export const hideFile = (fileId) => {
}) })
.then(resp => { .then(resp => {
if (resp.status == 200) { if (resp.status == 200) {
analytics().event('FILE.HIDE')
return return
} }
else { throw resp } else { throw resp }
@ -43,7 +42,6 @@ export const showFile = (fileId) => {
}) })
.then(resp => { .then(resp => {
if (resp.status == 200) { if (resp.status == 200) {
analytics().event('FILE.SHOW')
return return
} }
else { throw resp } else { throw resp }

4
FrontEnd/src/routes/SearchPage/modules/ImagePreview.js
Unescape Escape View File

@ -1,10 +1,6 @@
import { analytics } from 'utils'
export const TOGGLE_IMAGE_PREVIEW_MODAL = 'IMAGE_PREVIEW.TOGGLE_IMAGE_PREVIEW_MODAL' export const TOGGLE_IMAGE_PREVIEW_MODAL = 'IMAGE_PREVIEW.TOGGLE_IMAGE_PREVIEW_MODAL'
export const toggleImagePreview = (imageUrl) => { export const toggleImagePreview = (imageUrl) => {
analytics().event('IMAGE_PREVIEW.TOGGLE_IMAGE_PREIVEW')
return { return {
type: TOGGLE_IMAGE_PREVIEW_MODAL, type: TOGGLE_IMAGE_PREVIEW_MODAL,
imageUrl imageUrl

1
FrontEnd/src/routes/SearchPage/modules/SearchPage.js
Unescape Escape View File

@ -1,4 +1,3 @@
import { analytics } from 'utils'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout' import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { search, updateQuery } from 'routes/SearchPage/modules/SearchReducer' import { search, updateQuery } from 'routes/SearchPage/modules/SearchReducer'

4
FrontEnd/src/routes/SearchPage/modules/SearchReducer.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { stateValueExtractor, constants, titles, analytics } from 'utils/' import { stateValueExtractor, constants, titles } from 'utils/'
import { hitsModel, folderHitsModel } from 'models/' import { hitsModel, folderHitsModel } from 'models/'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout' import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout' import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -155,8 +155,6 @@ const performSearch = (page, query) => {
const clean = (page == 0) const clean = (page == 0)
dispatch(stopLoadingIndicator()) dispatch(stopLoadingIndicator())
dispatch(fillHits(clean, hits, data.found, query, hasMore, page)) dispatch(fillHits(clean, hits, data.found, query, hasMore, page))
if (page === 0) { analytics().event('SEARCH.PERFORM', { query: query }) }
}) })
.catch((errorPayload) => { .catch((errorPayload) => {
dispatch(stopLoadingIndicator()) dispatch(stopLoadingIndicator())

4
FrontEnd/src/routes/SearchPage/modules/TagsReducer.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { stateValueExtractor, constants, analytics } from 'utils/' import { stateValueExtractor, constants } from 'utils/'
import { hitsModel } from 'models/' import { hitsModel } from 'models/'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout' import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout' import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -53,7 +53,6 @@ export const addTagToFile = (fileId, tagType, tagName) => {
.then(resp => { .then(resp => {
if (resp.status == 200 || resp.status == 201) { if (resp.status == 200 || resp.status == 201) {
dispatch(markTagAsCreated(fileId, tagType, tagName)) dispatch(markTagAsCreated(fileId, tagType, tagName))
analytics().event('TAGS.ADD', { name: tagName })
return resp.json() return resp.json()
} }
else { throw resp } else { throw resp }
@ -81,7 +80,6 @@ export const removeTagFromFile = (fileId, tagType, tagName) => {
}) })
.then(resp => { .then(resp => {
if (resp.status == 200) { if (resp.status == 200) {
analytics().event('TAGS.REMOVED', { name: tagName })
return resp.json() return resp.json()
} }
else { throw resp } else { throw resp }

4
FrontEnd/src/routes/SearchPage/modules/UploadModal.js
Unescape Escape View File

@ -1,5 +1,5 @@
import { stateValueExtractor } from 'utils/' import { stateValueExtractor } from 'utils/'
import { analytics, FormDataPolyfill } from 'utils' import { FormDataPolyfill } from 'utils'
import { handleError, showInfo } from 'routes/CoreLayout/modules/CoreLayout' import { handleError, showInfo } from 'routes/CoreLayout/modules/CoreLayout'
import 'whatwg-fetch' import 'whatwg-fetch'
@ -41,7 +41,6 @@ export const uploadFiles = () => {
dispatch(toggleUploadModal()) dispatch(toggleUploadModal())
dispatch(cleanFilesToUpload()) dispatch(cleanFilesToUpload())
dispatch(showInfo('Files succesfully uploaded')) dispatch(showInfo('Files succesfully uploaded'))
analytics().event('SEARCH.UPLOAD_FILES', { count: uploadPromises.length })
}) })
.catch((errorPayload) => { .catch((errorPayload) => {
dispatch(filesUploading(false)) dispatch(filesUploading(false))
@ -50,7 +49,6 @@ export const uploadFiles = () => {
dispatch(handleError('No free space left in your account', true)) dispatch(handleError('No free space left in your account', true))
} else { } else {
dispatch(handleError(errorPayload)) dispatch(handleError(errorPayload))
analytics().event('SEARCH.UPLOAD_FILES_ERROR', { error: errorPayload })
} }
console.error('uploadFile', errorPayload) console.error('uploadFile', errorPayload)

3
FrontEnd/src/utils/index.js
Unescape Escape View File

@ -7,6 +7,5 @@ import * as stateValueExtractor from './stateValueExtractor'
import * as titles from './titles' import * as titles from './titles'
import * as constants from './constants' import * as constants from './constants'
import FormDataPolyfill from './formDataPolyfill' import FormDataPolyfill from './formDataPolyfill'
import analytics from './analytics'
export { validators, urls, dates, files, dom, stateValueExtractor, titles, constants, analytics, FormDataPolyfill } export { validators, urls, dates, files, dom, stateValueExtractor, titles, constants, FormDataPolyfill }

1
FrontEnd/src/utils/stateValueExtractor.js
Unescape Escape View File

@ -14,6 +14,7 @@ export const getDefaultSettings = () => {
return { return {
mode: 'cors', mode: 'cors',
credentials: 'include', credentials: 'include',
cache: 'no-cache',
headers: { headers: {
'Accept': 'application/json', 'Accept': 'application/json',
'Content-Type': 'application/json', 'Content-Type': 'application/json',

4
FrontEnd/src/utils/titles.js
Unescape Escape View File

@ -1,7 +1,3 @@
import analytics from './analytics'
export const setPageTitle = (title) => { export const setPageTitle = (title) => {
document.title = title document.title = title
analytics().register({ title: title })
analytics().event('LOCATION_CHANGED')
} }

25
LocalCrawler/src/index.js
Unescape Escape View File

@ -9,13 +9,18 @@ import cluster from 'cluster'
import 'babel-core/register' import 'babel-core/register'
import 'idempotent-babel-polyfill' import 'idempotent-babel-polyfill'
import { FileWatchService, ApiProxy } from './services' import { FileWatchService, ApiProxy, QueueProxy } from './services'
let app = null let app = null
if (cluster.isMaster) { if (cluster.isMaster) {
ApiProxy.logData(config.name, 'info', 'API runs on master thread') ApiProxy.logData(config.name, 'info', 'API runs on master thread')
ApiProxy.logData(config.name, 'info', 'Creating fork for the file-watcher process') ApiProxy.logData(config.name, 'info', 'Creating fork for the file-watcher process')
cluster.fork()
const worker = cluster.fork()
worker.on('exit', () => {
ApiProxy.logData(config.name, 'error', 'Worker thread crashed')
process.exit(1)
})
app = express() app = express()
app.server = http.createServer(app) app.server = http.createServer(app)
@ -34,16 +39,20 @@ if (cluster.isMaster) {
app.server.listen(process.env.PORT || config.port) app.server.listen(process.env.PORT || config.port)
console.log(`Started API on ${app.server.address().address}:${app.server.address().port}`) console.log(`Started API on ${app.server.address().address}:${app.server.address().port}`)
} else { } else {
ApiProxy.logData(config.name, 'info', 'File-watcher runs on worker thread') ApiProxy.logData(config.name, 'info', 'File-watcher runs on worker thread')
FileWatchService.startWatch() const rabbitErrorHandler = (error) => {
.catch(err => { ApiProxy.logData(config.name, 'error', `Rabbit Error: ${error}`)
ApiProxy.logData(config.name, 'error', `Error: ${err}`)
process.exit(1) process.exit(1)
}) }
QueueProxy.initRabbit(rabbitErrorHandler)
.then(() => FileWatchService.startWatch())
.catch(err => {
ApiProxy.logData(config.name, 'error', `Error: ${err}`)
process.exit(1)
})
} }
export default app export default app

27
LocalCrawler/src/services/FileWatchService.js
Unescape Escape View File

@ -8,24 +8,17 @@ import minimatch from 'minimatch'
import * as ApiProxy from './ApiProxy' import * as ApiProxy from './ApiProxy'
import * as QueueProxy from './QueueProxy' import * as QueueProxy from './QueueProxy'
export const startWatch = () => new Promise((resolve, reject) => { export const startWatch = () => {
QueueProxy.initRabbit() chokidar.watch(config.crawlPath, { usePolling: true, awaitWriteFinish: true })
.then(() => { .on('error', error => {
chokidar.watch(config.crawlPath, { usePolling: true, awaitWriteFinish: true }) ApiProxy.logData(config.name, 'error', `Chokidar error: ${error}`)
.on('error', error => {
ApiProxy.logData(config.name, 'error', `Chokidar error: ${error}`)
})
.on('all', (event, pathToFile, stat) => {
if (event === 'add' || event === 'change' || event === 'unlink') {
addTask(event, pathToFile, stat)
}
})
}) })
.catch(err => { .on('all', (event, pathToFile, stat) => {
ApiProxy.logData(config.name, 'error', `Error: ${err}`) if (event === 'add' || event === 'change' || event === 'unlink') {
reject(err) addTask(event, pathToFile, stat)
}
}) })
}) }
const shouldIgnore = (pathToFile, stat) => { const shouldIgnore = (pathToFile, stat) => {
if (!stat) { if (!stat) {
@ -82,7 +75,7 @@ const addTask = (event, pathToFile, stat) => {
const meta = { const meta = {
full_name: normalizedPath, full_name: normalizedPath,
updated_datetime: !stat ? '' : moment(stat.mtime).format('YYYY-MM-DD HH:mm:ss.SSS'), updated_datetime: !stat ? '' : moment(stat.mtime).format('YYYY-MM-DD HH:mm:ss.SSS'),
created_datetime: !stat ? '' :moment(stat.atime).format('YYYY-MM-DD HH:mm:ss.SSS'), created_datetime: !stat ? '' : moment(stat.atime).format('YYYY-MM-DD HH:mm:ss.SSS'),
source_id: config.name, source_id: config.name,
short_name: path.basename(normalizedPath), short_name: path.basename(normalizedPath),
extension: path.extname(normalizedPath), extension: path.extname(normalizedPath),

8
LocalCrawler/src/services/QueueProxy.js
Unescape Escape View File

@ -18,14 +18,10 @@ export const enqueueMessage = (message) => {
channel.publish(AMBAR_PIPELINE_EXCHANGE, '', Buffer.from(JSON.stringify(message)), { priority: priority }) channel.publish(AMBAR_PIPELINE_EXCHANGE, '', Buffer.from(JSON.stringify(message)), { priority: priority })
} }
export const initRabbit = () => new Promise((resolve, reject) => { export const initRabbit = (onError) => new Promise((resolve, reject) => {
amqp.connect(`${config.rabbitHost}?heartbeat=0`) amqp.connect(`${config.rabbitHost}?heartbeat=0`)
.then((conn) => { .then((conn) => {
conn.on('error', (err) => { conn.on('error', onError)
//eslint-disable-next-line no-console
console.error('Rabbit error!')
throw err
})
return conn.createChannel() return conn.createChannel()
.then(ch => { .then(ch => {

4
LocalCrawler/src/services/index.js
Unescape Escape View File

@ -1,7 +1,9 @@
import * as ApiProxy from './ApiProxy' import * as ApiProxy from './ApiProxy'
import * as FileWatchService from './FileWatchService' import * as FileWatchService from './FileWatchService'
import * as QueueProxy from './QueueProxy'
export { export {
ApiProxy, ApiProxy,
FileWatchService FileWatchService,
QueueProxy
} }

2
Pipeline/Dockerfile
Unescape Escape View File

@ -1,4 +1,4 @@
FROM python:3 FROM python:3.6.6-stretch
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
curl \ curl \

BIN
Pipeline/requirements.txt
View File

Binary file not shown.

3
Rabbit/Dockerfile
Unescape Escape View File

@ -1,4 +1,4 @@
FROM rabbitmq:3-management FROM rabbitmq:3.6-management
RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl
@ -7,6 +7,7 @@ ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
COPY rabbitmq.config /etc/rabbitmq/rabbitmq.config COPY rabbitmq.config /etc/rabbitmq/rabbitmq.config
RUN chmod 777 /etc/rabbitmq/rabbitmq.config
HEALTHCHECK --interval=5s --timeout=30s --retries=50 \ HEALTHCHECK --interval=5s --timeout=30s --retries=50 \
CMD curl -f localhost:15672 || exit 1 CMD curl -f localhost:15672 || exit 1

32
ServiceApi/src/services/EsProxy/AmbarFileDataMapping.json
Unescape Escape View File

@ -5,18 +5,21 @@
"index.mapper.dynamic": false, "index.mapper.dynamic": false,
"analysis": { "analysis": {
"char_filter": { "char_filter": {
"russian_cf": { "ambar_cf": {
"type": "mapping", "type": "mapping",
"mappings": [ "mappings": [
"ё => е", "ё => е",
"Ё => Е", "Ё => Е",
"Й => И",
"й => и",
": => \\u0020" ": => \\u0020"
] ]
} }
}, },
"filter": { "filter": {
"russian_replace_i": {
"type": "pattern_replace",
"pattern": "й",
"replacement": "и"
},
"russian_stop": { "russian_stop": {
"type": "stop", "type": "stop",
"stopwords": "_russian_" "stopwords": "_russian_"
@ -25,10 +28,6 @@
"type": "stemmer", "type": "stemmer",
"language": "russian" "language": "russian"
}, },
"russian_stop_alt": {
"type": "stop",
"stopwords": "а,без,более,бы,был,была,были,было,быть,в,вам,вас,весь,во,вот,все,всего,всех,вы,где,да,даже,для,до,его,ее,если,есть,еще,же,за,здесь,и,из,или,им,их,к,как,ко,когда,кто,ли,либо,мне,может,мы,на,надо,наш,не,него,нее,нет,ни,них,но,ну,о,об,однако,он,она,они,оно,от,очень,по,под,при,с,со,так,также,такой,там,те,тем,то,того,тоже,той,только,том,ты,у,уже,хотя,чего,чей,чем,что,чтобы,чье,чья,эта,эти,это,я,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"
},
"ambar_word_delimiter": { "ambar_word_delimiter": {
"type": "word_delimiter", "type": "word_delimiter",
"generate_word_parts": true, "generate_word_parts": true,
@ -102,29 +101,18 @@
"lowercase" "lowercase"
] ]
}, },
"ambar_ru_old": {
"tokenizer": "standard",
"char_filter": [
"russian_cf"
],
"filter": [
"lowercase",
"russian_stop",
"russian_stemmer",
"ambar_word_delimiter"
]
},
"ambar_ru": { "ambar_ru": {
"tokenizer": "standard", "tokenizer": "standard",
"char_filter": [ "char_filter": [
"russian_cf" "ambar_cf"
], ],
"filter": [ "filter": [
"lowercase", "lowercase",
"russian_stop",
"russian_morphology", "russian_morphology",
"english_morphology", "english_morphology",
"russian_stop_alt", "ambar_word_delimiter",
"ambar_word_delimiter" "russian_replace_i"
] ]
}, },
"ambar_en": { "ambar_en": {

6
ServiceApi/src/services/QueueProxy.js
Unescape Escape View File

@ -32,7 +32,7 @@ export const enqueuePipelineMessage = (storage, message) => new Promise((resolve
}) })
export const initRabbit = new Promise((resolve, reject) => { export const initRabbit = new Promise((resolve, reject) => {
amqp.connect(`${config.rabbitHost}?heartbeat=60`) amqp.connect(`${config.rabbitHost}?heartbeat=0`)
.then((conn) => { .then((conn) => {
conn.on('error', (err) => { conn.on('error', (err) => {
//eslint-disable-next-line no-console //eslint-disable-next-line no-console
@ -45,9 +45,9 @@ export const initRabbit = new Promise((resolve, reject) => {
.then(() => channel.assertExchange(AMBAR_PIPELINE_WAITING_EXCHANGE, .then(() => channel.assertExchange(AMBAR_PIPELINE_WAITING_EXCHANGE,
'fanout', { durable: false })) 'fanout', { durable: false }))
.then(() => channel.assertQueue(AMBAR_PIPELINE_QUEUE, .then(() => channel.assertQueue(AMBAR_PIPELINE_QUEUE,
{ durable: false, arguments: { 'x-dead-letter-exchange': AMBAR_PIPELINE_WAITING_EXCHANGE, 'x-max-priority': AMBAR_PIPELINE_QUEUE_MAX_PRIORITY } })) { durable: false, arguments: { 'x-queue-mode': 'lazy', 'x-dead-letter-exchange': AMBAR_PIPELINE_WAITING_EXCHANGE, 'x-max-priority': AMBAR_PIPELINE_QUEUE_MAX_PRIORITY } }))
.then(() => channel.assertQueue(AMBAR_PIPELINE_WAITING_QUEUE, .then(() => channel.assertQueue(AMBAR_PIPELINE_WAITING_QUEUE,
{ durable: false, arguments: { 'x-dead-letter-exchange': AMBAR_PIPELINE_EXCHANGE, 'x-message-ttl': AMBAR_PIPELINE_WAITING_QUEUE_TTL } })) { durable: false, arguments: { 'x-queue-mode': 'lazy', 'x-dead-letter-exchange': AMBAR_PIPELINE_EXCHANGE, 'x-message-ttl': AMBAR_PIPELINE_WAITING_QUEUE_TTL } }))
.then(() => channel.bindQueue(AMBAR_PIPELINE_QUEUE, .then(() => channel.bindQueue(AMBAR_PIPELINE_QUEUE,
AMBAR_PIPELINE_EXCHANGE)) AMBAR_PIPELINE_EXCHANGE))
.then(() => channel.bindQueue(AMBAR_PIPELINE_WAITING_QUEUE, .then(() => channel.bindQueue(AMBAR_PIPELINE_WAITING_QUEUE,

2
WebApi/package.json
Unescape Escape View File

@ -1,6 +1,6 @@
{ {
"name": "ambar-webapi", "name": "ambar-webapi",
"version": "2.1.0", "version": "2.1.18",
"description": "Ambar WebAPI", "description": "Ambar WebAPI",
"main": "dist", "main": "dist",
"scripts": { "scripts": {

1
WebApi/src/api/index.js
Unescape Escape View File

@ -20,7 +20,6 @@ export default ({ config, storage }) => {
api.get('/', (req, res) => { api.get('/', (req, res) => {
res.json({ res.json({
version: version, version: version,
analyticsToken: config.analyticsToken,
uiLang: config.uiLang, uiLang: config.uiLang,
rawConfig: config rawConfig: config
}) })

1
WebApi/src/config.js
Unescape Escape View File

@ -12,7 +12,6 @@ const defaultConfig = {
"serviceApiUrl": "http://localhost:8081", "serviceApiUrl": "http://localhost:8081",
"rabbitHost": "amqp://ambar", "rabbitHost": "amqp://ambar",
"uiLang": "en", "uiLang": "en",
"analyticsToken": "",
"crawlerPort": 8082 "crawlerPort": 8082
} }

2
WebApi/src/services/QueueProxy.js
Unescape Escape View File

@ -26,7 +26,7 @@ export const enqueuePipelineMessage = (storage, message) => new Promise((resolve
}) })
export const initRabbit = new Promise((resolve, reject) => { export const initRabbit = new Promise((resolve, reject) => {
amqp.connect(`${config.rabbitHost}?heartbeat=60`) amqp.connect(`${config.rabbitHost}?heartbeat=0`)
.then((conn) => { .then((conn) => {
conn.on('error', (err) => { conn.on('error', (err) => {
//eslint-disable-next-line no-console //eslint-disable-next-line no-console

Write Preview
Loading…
Cancel
Save