abderr
/
ambar2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Version 2.1.18

Browse Source
master v2.1.18
Ilya.P 8 years ago
parent eae54a9750
commit a143182063
28 changed files with 94 additions and 135 deletions
Show Stats Download Patch File Download Diff File

10
FrontEnd/src/components/Search/components/SearchInput/SearchInput.js
Unescape Escape View File

@ -7,8 +7,6 @@ import MediaQuery from 'react-responsive'
import classes from './SearchInput.scss'
class SearchInput extends Component {
static timeoutId = null
componentDidMount() {
this.refs.search_input.focus()
}
@ -50,16 +48,8 @@ class SearchInput extends Component {
return
}
}}
onKeyDown={(event) => {
clearTimeout(this.timeoutId)
}}
onChange={(event, newValue) => {
clearTimeout(this.timeoutId)
setQuery(newValue)
this.timeoutId = setTimeout(() => {
search(0, newValue)
}, 500)
}}
underlineShow={false}
/>

9
FrontEnd/src/index.html
Unescape Escape View File

@ -1,5 +1,6 @@
<!doctype html>
<html lang="en">
<head>
<title>Loading...</title>
<meta charset="utf-8">
@ -12,14 +13,10 @@
<meta name="apple-mobile-web-app-title" content="Ambar">
<meta name="application-name" content="Ambar">
<meta name="theme-color" content="#ffffff">
<!-- start Mixpanel -->
<script type="text/javascript">(function(e,a){if(!a.__SV){var b=window;try{var c,l,i,j=b.location,g=j.hash;c=function(a,b){return(l=a.match(RegExp(b+"=([^&]*)")))?l[1]:null};g&&c(g,"state")&&(i=JSON.parse(decodeURIComponent(c(g,"state"))),"mpeditor"===i.action&&(b.sessionStorage.setItem("_mpcehash",g),history.replaceState(i.desiredHash||"",e.title,j.pathname+j.search)))}catch(m){}var k,h;window.mixpanel=a;a._i=[];a.init=function(b,c,f){function e(b,a){var c=a.split(".");2==c.length&&(b=b[c[0]],a=c[1]);b[a]=function(){b.push([a].concat(Array.prototype.slice.call(arguments,
0)))}}var d=a;"undefined"!==typeof f?d=a[f]=[]:f="mixpanel";d.people=d.people||[];d.toString=function(b){var a="mixpanel";"mixpanel"!==f&&(a+="."+f);b||(a+=" (stub)");return a};d.people.toString=function(){return d.toString(1)+".people (stub)"};k="disable time_event track track_pageview track_links track_forms register register_once alias unregister identify name_tag set_config reset people.set people.set_once people.increment people.append people.union people.track_charge people.clear_charges people.delete_user".split(" ");
for(h=0;h<k.length;h++)e(d,k[h]);a._i.push([b,c,f])};a.__SV=1.2;b=e.createElement("script");b.type="text/javascript";b.async=!0;b.src="undefined"!==typeof MIXPANEL_CUSTOM_LIB_URL?MIXPANEL_CUSTOM_LIB_URL:"file:"===e.location.protocol&&"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js".match(/^\/\//)?"https://cdn.mxpnl.com/libs/mixpanel-2-latest.min.js":"//cdn.mxpnl.com/libs/mixpanel-2-latest.min.js";c=e.getElementsByTagName("script")[0];c.parentNode.insertBefore(b,c)}})(document,window.mixpanel||[]);
</script>
<!-- end Mixpanel -->
</head>
<body>
<div id="root" style="height: 100%"></div>
</body>
</html>

21
FrontEnd/src/routes/CoreLayout/modules/CoreLayout.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { titles, stateValueExtractor, urls, constants, analytics } from 'utils'
import { titles, stateValueExtractor, urls, constants } from 'utils'
import 'whatwg-fetch'
const CHANGE_FIELD = 'CORE.CHANGE_FIELD'
@ -24,9 +24,6 @@ export const loadConfig = () => {
dispatch(changeField('version', apiInfo.version))
dispatch(changeField('lang', apiInfo.uiLang))
analytics(apiInfo.analyticsToken)
analytics().register({ apiUrl: urls.apiHost })
})
.then(() => dispatch(stopLoadingIndicator()))
.catch(error => {
@ -56,7 +53,10 @@ const stopLoadingIndicator = () => {
const getApiUrl = () => new Promise((resolve, reject) => {
fetch('apiUrl.txt', {
method: 'GET'
method: 'GET',
mode: 'cors',
credentials: 'include',
cache: 'no-cache'
})
.then(resp => resolve(resp.text()))
.catch(err => reject(err))
@ -64,7 +64,10 @@ const getApiUrl = () => new Promise((resolve, reject) => {
const getLocalizationsJson = () => new Promise((resolve, reject) => {
fetch('localizations.json', {
method: 'GET'
method: 'GET',
mode: 'cors',
credentials: 'include',
cache: 'no-cache'
})
.then(resp => resolve(resp.text()))
.catch(err => reject(err))
@ -72,7 +75,10 @@ const getLocalizationsJson = () => new Promise((resolve, reject) => {
const getWebApiInfo = (url) => new Promise((resolve, reject) => {
fetch(url, {
method: 'GET'
method: 'GET',
mode: 'cors',
credentials: 'include',
cache: 'no-cache'
})
.then(resp => resolve(resp.json()))
.catch(err => reject(err))
@ -89,7 +95,6 @@ export function handleError(error, showErrorMessage = false) {
if (error.constructor === Response) {
error = `Response ${error.status} ${error.statusText} at ${error.url}`
}
analytics().event('ERROR', { description: error ? error.toString() : 'no info' })
console.log(error)
return {

3
FrontEnd/src/routes/MainLayout/modules/MainLayout.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { urls, titles, stateValueExtractor, analytics } from 'utils'
import { urls, titles, stateValueExtractor } from 'utils'
import { push } from 'react-router-redux'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
@ -44,7 +44,6 @@ export function stopLoadingIndicator() {
export const toggleRateUsModal = (value) => {
return (dispatch, getState) => {
dispatch(changeField('showRateUsModal', value))
analytics().event('ACCOUNT.RATE_US_MODAL_OPENED')
}
}

2
FrontEnd/src/routes/SearchPage/modules/DetailedView.js
Unescape Escape View File

@ -1,6 +1,5 @@
import { stateValueExtractor } from 'utils/'
import { hitsModel } from 'models/'
import { analytics } from 'utils'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -25,7 +24,6 @@ export const loadHighlight = (fileId, query) => {
.then((resp) => {
dispatch(setContentHighlight(fileId, hitsModel.contentHighlightFromApi(resp)))
dispatch(startStopHighlightLoadingIndicator(fileId, false))
analytics().event('SEARCH.LOAD_HIGHLIGHT')
})
.catch((errorPayload) => {
dispatch(startStopHighlightLoadingIndicator(fileId, false))

4
FrontEnd/src/routes/SearchPage/modules/FileVisibilityReducer.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { stateValueExtractor, analytics } from 'utils'
import { stateValueExtractor } from 'utils'
import { hitsModel } from 'models/'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -18,7 +18,6 @@ export const hideFile = (fileId) => {
})
.then(resp => {
if (resp.status == 200) {
analytics().event('FILE.HIDE')
return
}
else { throw resp }
@ -43,7 +42,6 @@ export const showFile = (fileId) => {
})
.then(resp => {
if (resp.status == 200) {
analytics().event('FILE.SHOW')
return
}
else { throw resp }

4
FrontEnd/src/routes/SearchPage/modules/ImagePreview.js
Unescape Escape View File

@ -1,10 +1,6 @@
import { analytics } from 'utils'
export const TOGGLE_IMAGE_PREVIEW_MODAL = 'IMAGE_PREVIEW.TOGGLE_IMAGE_PREVIEW_MODAL'
export const toggleImagePreview = (imageUrl) => {
analytics().event('IMAGE_PREVIEW.TOGGLE_IMAGE_PREIVEW')
return {
type: TOGGLE_IMAGE_PREVIEW_MODAL,
imageUrl

1
FrontEnd/src/routes/SearchPage/modules/SearchPage.js
Unescape Escape View File

@ -1,4 +1,3 @@
import { analytics } from 'utils'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { search, updateQuery } from 'routes/SearchPage/modules/SearchReducer'

4
FrontEnd/src/routes/SearchPage/modules/SearchReducer.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { stateValueExtractor, constants, titles, analytics } from 'utils/'
import { stateValueExtractor, constants, titles } from 'utils/'
import { hitsModel, folderHitsModel } from 'models/'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -155,8 +155,6 @@ const performSearch = (page, query) => {
const clean = (page == 0)
dispatch(stopLoadingIndicator())
dispatch(fillHits(clean, hits, data.found, query, hasMore, page))
if (page === 0) { analytics().event('SEARCH.PERFORM', { query: query }) }
})
.catch((errorPayload) => {
dispatch(stopLoadingIndicator())

4
FrontEnd/src/routes/SearchPage/modules/TagsReducer.js
Unescape Escape View File

@ -1,4 +1,4 @@
import { stateValueExtractor, constants, analytics } from 'utils/'
import { stateValueExtractor, constants } from 'utils/'
import { hitsModel } from 'models/'
import { handleError } from 'routes/CoreLayout/modules/CoreLayout'
import { startLoadingIndicator, stopLoadingIndicator } from 'routes/MainLayout/modules/MainLayout'
@ -53,7 +53,6 @@ export const addTagToFile = (fileId, tagType, tagName) => {
.then(resp => {
if (resp.status == 200 || resp.status == 201) {
dispatch(markTagAsCreated(fileId, tagType, tagName))
analytics().event('TAGS.ADD', { name: tagName })
return resp.json()
}
else { throw resp }
@ -81,7 +80,6 @@ export const removeTagFromFile = (fileId, tagType, tagName) => {
})
.then(resp => {
if (resp.status == 200) {
analytics().event('TAGS.REMOVED', { name: tagName })
return resp.json()
}
else { throw resp }

4
FrontEnd/src/routes/SearchPage/modules/UploadModal.js
Unescape Escape View File

@ -1,5 +1,5 @@
import { stateValueExtractor } from 'utils/'
import { analytics, FormDataPolyfill } from 'utils'
import { FormDataPolyfill } from 'utils'
import { handleError, showInfo } from 'routes/CoreLayout/modules/CoreLayout'
import 'whatwg-fetch'
@ -41,7 +41,6 @@ export const uploadFiles = () => {
dispatch(toggleUploadModal())
dispatch(cleanFilesToUpload())
dispatch(showInfo('Files succesfully uploaded'))
analytics().event('SEARCH.UPLOAD_FILES', { count: uploadPromises.length })
})
.catch((errorPayload) => {
dispatch(filesUploading(false))
@ -50,7 +49,6 @@ export const uploadFiles = () => {
dispatch(handleError('No free space left in your account', true))
} else {
dispatch(handleError(errorPayload))
analytics().event('SEARCH.UPLOAD_FILES_ERROR', { error: errorPayload })
}
console.error('uploadFile', errorPayload)

3
FrontEnd/src/utils/index.js
Unescape Escape View File

@ -7,6 +7,5 @@ import * as stateValueExtractor from './stateValueExtractor'
import * as titles from './titles'
import * as constants from './constants'
import FormDataPolyfill from './formDataPolyfill'
import analytics from './analytics'
export { validators, urls, dates, files, dom, stateValueExtractor, titles, constants, analytics, FormDataPolyfill }
export { validators, urls, dates, files, dom, stateValueExtractor, titles, constants, FormDataPolyfill }

1
FrontEnd/src/utils/stateValueExtractor.js
Unescape Escape View File

@ -14,6 +14,7 @@ export const getDefaultSettings = () => {
return {
mode: 'cors',
credentials: 'include',
cache: 'no-cache',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json',

4
FrontEnd/src/utils/titles.js
Unescape Escape View File

@ -1,7 +1,3 @@
import analytics from './analytics'
export const setPageTitle = (title) => {
document.title = title
analytics().register({ title: title })
analytics().event('LOCATION_CHANGED')
}

25
LocalCrawler/src/index.js
Unescape Escape View File

@ -9,13 +9,18 @@ import cluster from 'cluster'
import 'babel-core/register'
import 'idempotent-babel-polyfill'
import { FileWatchService, ApiProxy } from './services'
import { FileWatchService, ApiProxy, QueueProxy } from './services'
let app = null
if (cluster.isMaster) {
ApiProxy.logData(config.name, 'info', 'API runs on master thread')
ApiProxy.logData(config.name, 'info', 'Creating fork for the file-watcher process')
cluster.fork()
const worker = cluster.fork()
worker.on('exit', () => {
ApiProxy.logData(config.name, 'error', 'Worker thread crashed')
process.exit(1)
})
app = express()
app.server = http.createServer(app)
@ -34,16 +39,20 @@ if (cluster.isMaster) {
app.server.listen(process.env.PORT || config.port)
console.log(`Started API on ${app.server.address().address}:${app.server.address().port}`)
} else {
ApiProxy.logData(config.name, 'info', 'File-watcher runs on worker thread')
FileWatchService.startWatch()
.catch(err => {
ApiProxy.logData(config.name, 'error', `Error: ${err}`)
const rabbitErrorHandler = (error) => {
ApiProxy.logData(config.name, 'error', `Rabbit Error: ${error}`)
process.exit(1)
})
}
QueueProxy.initRabbit(rabbitErrorHandler)
.then(() => FileWatchService.startWatch())
.catch(err => {
ApiProxy.logData(config.name, 'error', `Error: ${err}`)
process.exit(1)
})
}
export default app

27
LocalCrawler/src/services/FileWatchService.js
Unescape Escape View File

@ -8,24 +8,17 @@ import minimatch from 'minimatch'
import * as ApiProxy from './ApiProxy'
import * as QueueProxy from './QueueProxy'
export const startWatch = () => new Promise((resolve, reject) => {
QueueProxy.initRabbit()
.then(() => {
chokidar.watch(config.crawlPath, { usePolling: true, awaitWriteFinish: true })
.on('error', error => {
ApiProxy.logData(config.name, 'error', `Chokidar error: ${error}`)
})
.on('all', (event, pathToFile, stat) => {
if (event === 'add' || event === 'change' || event === 'unlink') {
addTask(event, pathToFile, stat)
}
})
export const startWatch = () => {
chokidar.watch(config.crawlPath, { usePolling: true, awaitWriteFinish: true })
.on('error', error => {
ApiProxy.logData(config.name, 'error', `Chokidar error: ${error}`)
})
.catch(err => {
ApiProxy.logData(config.name, 'error', `Error: ${err}`)
reject(err)
.on('all', (event, pathToFile, stat) => {
if (event === 'add' || event === 'change' || event === 'unlink') {
addTask(event, pathToFile, stat)
}
})
})
}
const shouldIgnore = (pathToFile, stat) => {
if (!stat) {
@ -82,7 +75,7 @@ const addTask = (event, pathToFile, stat) => {
const meta = {
full_name: normalizedPath,
updated_datetime: !stat ? '' : moment(stat.mtime).format('YYYY-MM-DD HH:mm:ss.SSS'),
created_datetime: !stat ? '' :moment(stat.atime).format('YYYY-MM-DD HH:mm:ss.SSS'),
created_datetime: !stat ? '' : moment(stat.atime).format('YYYY-MM-DD HH:mm:ss.SSS'),
source_id: config.name,
short_name: path.basename(normalizedPath),
extension: path.extname(normalizedPath),

8
LocalCrawler/src/services/QueueProxy.js
Unescape Escape View File

@ -18,14 +18,10 @@ export const enqueueMessage = (message) => {
channel.publish(AMBAR_PIPELINE_EXCHANGE, '', Buffer.from(JSON.stringify(message)), { priority: priority })
}
export const initRabbit = () => new Promise((resolve, reject) => {
export const initRabbit = (onError) => new Promise((resolve, reject) => {
amqp.connect(`${config.rabbitHost}?heartbeat=0`)
.then((conn) => {
conn.on('error', (err) => {
//eslint-disable-next-line no-console
console.error('Rabbit error!')
throw err
})
conn.on('error', onError)
return conn.createChannel()
.then(ch => {

4
LocalCrawler/src/services/index.js
Unescape Escape View File

@ -1,7 +1,9 @@
import * as ApiProxy from './ApiProxy'
import * as FileWatchService from './FileWatchService'
import * as QueueProxy from './QueueProxy'
export {
ApiProxy,
FileWatchService
FileWatchService,
QueueProxy
}

2
Pipeline/Dockerfile
Unescape Escape View File

@ -1,4 +1,4 @@
FROM python:3
FROM python:3.6.6-stretch
RUN apt-get update && apt-get install -y \
curl \

BIN
Pipeline/requirements.txt
View File

Binary file not shown.

3
Rabbit/Dockerfile
Unescape Escape View File

@ -1,4 +1,4 @@
FROM rabbitmq:3-management
FROM rabbitmq:3.6-management
RUN apt-get update && apt-get install --no-install-recommends --no-install-suggests -y curl
@ -7,6 +7,7 @@ ENV TZ=UTC
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
COPY rabbitmq.config /etc/rabbitmq/rabbitmq.config
RUN chmod 777 /etc/rabbitmq/rabbitmq.config
HEALTHCHECK --interval=5s --timeout=30s --retries=50 \
CMD curl -f localhost:15672 || exit 1

32
ServiceApi/src/services/EsProxy/AmbarFileDataMapping.json
Unescape Escape View File

@ -5,18 +5,21 @@
"index.mapper.dynamic": false,
"analysis": {
"char_filter": {
"russian_cf": {
"ambar_cf": {
"type": "mapping",
"mappings": [
"ё => е",
"Ё => Е",
"Й => И",
"й => и",
": => \\u0020"
]
}
},
"filter": {
"russian_replace_i": {
"type": "pattern_replace",
"pattern": "й",
"replacement": "и"
},
"russian_stop": {
"type": "stop",
"stopwords": "_russian_"
@ -25,10 +28,6 @@
"type": "stemmer",
"language": "russian"
},
"russian_stop_alt": {
"type": "stop",
"stopwords": "а,без,более,бы,был,была,были,было,быть,в,вам,вас,весь,во,вот,все,всего,всех,вы,где,да,даже,для,до,его,ее,если,есть,еще,же,за,здесь,и,из,или,им,их,к,как,ко,когда,кто,ли,либо,мне,может,мы,на,надо,наш,не,него,нее,нет,ни,них,но,ну,о,об,однако,он,она,они,оно,от,очень,по,под,при,с,со,так,также,такой,там,те,тем,то,того,тоже,той,только,том,ты,у,уже,хотя,чего,чей,чем,что,чтобы,чье,чья,эта,эти,это,я,a,an,and,are,as,at,be,but,by,for,if,in,into,is,it,no,not,of,on,or,such,that,the,their,then,there,these,they,this,to,was,will,with"
},
"ambar_word_delimiter": {
"type": "word_delimiter",
"generate_word_parts": true,
@ -102,29 +101,18 @@
"lowercase"
]
},
"ambar_ru_old": {
"tokenizer": "standard",
"char_filter": [
"russian_cf"
],
"filter": [
"lowercase",
"russian_stop",
"russian_stemmer",
"ambar_word_delimiter"
]
},
"ambar_ru": {
"tokenizer": "standard",
"char_filter": [
"russian_cf"
"ambar_cf"
],
"filter": [
"lowercase",
"russian_stop",
"russian_morphology",
"english_morphology",
"russian_stop_alt",
"ambar_word_delimiter"
"ambar_word_delimiter",
"russian_replace_i"
]
},
"ambar_en": {

6
ServiceApi/src/services/QueueProxy.js
Unescape Escape View File

@ -32,7 +32,7 @@ export const enqueuePipelineMessage = (storage, message) => new Promise((resolve
})
export const initRabbit = new Promise((resolve, reject) => {
amqp.connect(`${config.rabbitHost}?heartbeat=60`)
amqp.connect(`${config.rabbitHost}?heartbeat=0`)
.then((conn) => {
conn.on('error', (err) => {
//eslint-disable-next-line no-console
@ -45,9 +45,9 @@ export const initRabbit = new Promise((resolve, reject) => {
.then(() => channel.assertExchange(AMBAR_PIPELINE_WAITING_EXCHANGE,
'fanout', { durable: false }))
.then(() => channel.assertQueue(AMBAR_PIPELINE_QUEUE,
{ durable: false, arguments: { 'x-dead-letter-exchange': AMBAR_PIPELINE_WAITING_EXCHANGE, 'x-max-priority': AMBAR_PIPELINE_QUEUE_MAX_PRIORITY } }))
{ durable: false, arguments: { 'x-queue-mode': 'lazy', 'x-dead-letter-exchange': AMBAR_PIPELINE_WAITING_EXCHANGE, 'x-max-priority': AMBAR_PIPELINE_QUEUE_MAX_PRIORITY } }))
.then(() => channel.assertQueue(AMBAR_PIPELINE_WAITING_QUEUE,
{ durable: false, arguments: { 'x-dead-letter-exchange': AMBAR_PIPELINE_EXCHANGE, 'x-message-ttl': AMBAR_PIPELINE_WAITING_QUEUE_TTL } }))
{ durable: false, arguments: { 'x-queue-mode': 'lazy', 'x-dead-letter-exchange': AMBAR_PIPELINE_EXCHANGE, 'x-message-ttl': AMBAR_PIPELINE_WAITING_QUEUE_TTL } }))
.then(() => channel.bindQueue(AMBAR_PIPELINE_QUEUE,
AMBAR_PIPELINE_EXCHANGE))
.then(() => channel.bindQueue(AMBAR_PIPELINE_WAITING_QUEUE,

2
WebApi/package.json
Unescape Escape View File

@ -1,6 +1,6 @@
{
"name": "ambar-webapi",
"version": "2.1.0",
"version": "2.1.18",
"description": "Ambar WebAPI",
"main": "dist",
"scripts": {

1
WebApi/src/api/index.js
Unescape Escape View File

@ -20,7 +20,6 @@ export default ({ config, storage }) => {
api.get('/', (req, res) => {
res.json({
version: version,
analyticsToken: config.analyticsToken,
uiLang: config.uiLang,
rawConfig: config
})

1
WebApi/src/config.js
Unescape Escape View File

@ -12,7 +12,6 @@ const defaultConfig = {
"serviceApiUrl": "http://localhost:8081",
"rabbitHost": "amqp://ambar",
"uiLang": "en",
"analyticsToken": "",
"crawlerPort": 8082
}

2
WebApi/src/services/QueueProxy.js
Unescape Escape View File

@ -26,7 +26,7 @@ export const enqueuePipelineMessage = (storage, message) => new Promise((resolve
})
export const initRabbit = new Promise((resolve, reject) => {
amqp.connect(`${config.rabbitHost}?heartbeat=60`)
amqp.connect(`${config.rabbitHost}?heartbeat=0`)
.then((conn) => {
conn.on('error', (err) => {
//eslint-disable-next-line no-console

Write Preview
Loading…
Cancel
Save