From 68621b66d840a9c21d9d3eb2bd243a02719c586b Mon Sep 17 00:00:00 2001 From: Orzu Ionut Date: Mon, 31 May 2021 15:06:43 +0300 Subject: [PATCH] Fix issue with handling PDF files. Fix issue with running the operations from the command line. --- app/Console/Commands/RunSearchDisplace.php | 66 +++++++++++++++++-- app/Http/Controllers/FileController.php | 6 +- app/SearchDisplace/Ingest/SendDocument.php | 10 +-- .../SearchAndDisplaceFromFiles.php | 21 +++--- app/SearchDisplace/Webhooks/HandleRequest.php | 1 - public/js/app.js | 63 ++++++++++++++++-- .../js/components/ProcessFile/ProcessFile.ts | 35 ++++++---- .../js/components/ProcessFile/ProcessFile.vue | 27 +++++--- 8 files changed, 180 insertions(+), 49 deletions(-) diff --git a/app/Console/Commands/RunSearchDisplace.php b/app/Console/Commands/RunSearchDisplace.php index 06cfe00..8b0ca22 100644 --- a/app/Console/Commands/RunSearchDisplace.php +++ b/app/Console/Commands/RunSearchDisplace.php @@ -44,13 +44,20 @@ class RunSearchDisplace extends Command $searchers = $this->argument('filters'); $id = md5(uniqid(rand(), true)); + $pathDetails = pathinfo($documentPath); - $this->storeSearchers($id, $searchers, $documentPath); + $resultedDocumentPath = $pathDetails['dirname'] . '/' . $pathDetails['filename'] . '-displaced.md'; + + $this->storeSearchers($id, $searchers, $resultedDocumentPath); try { $sendToIngest = new SendDocument(); - $sendToIngest->execute($documentPath, $id); + $sendToIngest->execute($id, [ + 'path' => $documentPath, + 'name' => $pathDetails['basename'], + 'type' => $this->getFileMimeType($documentPath), + ]); $this->info('Processing document..'); $this->info('After the processing will be done the result will show up at the same path as the input.'); @@ -59,11 +66,12 @@ class RunSearchDisplace extends Command } } - protected function storeSearchers($id, $searchers, $documentPath) + protected function storeSearchers($id, $searchers, $storeResultPath) { $data = [ - 'searchers' => $this->getSearchers($searchers), - 'document_path' => $documentPath, + 'searchers' => $this->getSearchers($searchers), + + 'document_path' => $storeResultPath, ]; $storage = Storage::disk('local'); @@ -95,14 +103,58 @@ class RunSearchDisplace extends Command return $list; } - protected function getSearchersFromFile($path) + protected function getSearchersFromFile($argument) { + $result = explode(':', $argument); + + if (count($result) > 1) { + $path = $result[0]; + + $replaceWith = $result[1]; + } else { + $path = $argument; + $replaceWith = ''; + } + $contents = file_get_contents($path); if ( ! $contents) { throw new \Exception('Something went wrong when tried reading from file.'); } - return json_decode($contents); + return [ + [ + 'content' => json_decode($contents), + 'replace_with' => $replaceWith, + ], + ]; + } + + protected function getFileMimeType($file) { + if (function_exists('finfo_file')) { + $finfo = finfo_open(FILEINFO_MIME_TYPE); + $type = finfo_file($finfo, $file); + finfo_close($finfo); + } else { + require_once 'upgradephp/ext/mime.php'; + $type = mime_content_type($file); + } + + if (!$type || in_array($type, array('application/octet-stream', 'text/plain'))) { + $secondOpinion = exec('file -b --mime-type ' . escapeshellarg($file), $foo, $returnCode); + if ($returnCode === 0 && $secondOpinion) { + $type = $secondOpinion; + } + } + + if (!$type || in_array($type, array('application/octet-stream', 'text/plain'))) { + require_once 'upgradephp/ext/mime.php'; + $exifImageType = exif_imagetype($file); + if ($exifImageType !== false) { + $type = image_type_to_mime_type($exifImageType); + } + } + + return $type; } } diff --git a/app/Http/Controllers/FileController.php b/app/Http/Controllers/FileController.php index 9cc1129..9421c0d 100644 --- a/app/Http/Controllers/FileController.php +++ b/app/Http/Controllers/FileController.php @@ -26,7 +26,11 @@ class FileController extends Controller $sendDocument = new SendDocument(); - $sendDocument->execute($file->getRealPath(), $fileId); + $sendDocument->execute($fileId, [ + 'path' => $file->getRealPath(), + 'type' => $file->getMimeType(), + 'name' => $file->getClientOriginalName() + ]); return response()->json([ 'id' => $fileId, diff --git a/app/SearchDisplace/Ingest/SendDocument.php b/app/SearchDisplace/Ingest/SendDocument.php index 7a6615b..72a7c00 100644 --- a/app/SearchDisplace/Ingest/SendDocument.php +++ b/app/SearchDisplace/Ingest/SendDocument.php @@ -13,10 +13,10 @@ class SendDocument $this->url = env('SD_INGEST_URL') . '/ingest'; } - public function execute($documentPath, $id) + public function execute($id, $document) { try { - $response = $this->sendRequest($documentPath, $id); + $response = $this->sendRequest($id, $document); if ($response['status'] === 'fail') { $message = array_key_exists('message', $response) @@ -32,7 +32,7 @@ class SendDocument } } - public function sendRequest($documentPath, $id) + public function sendRequest($id, $document) { $client = new Client(); @@ -49,7 +49,9 @@ class SendDocument [ 'name' => 'document', - 'contents' => fopen($documentPath, 'r'), + 'contents' => fopen($document['path'], 'r'), + 'filename' => $document['name'], + 'Content-type' => $document['type'], ] ], ]); diff --git a/app/SearchDisplace/SearchAndDisplaceFromFiles.php b/app/SearchDisplace/SearchAndDisplaceFromFiles.php index f813325..cb52eeb 100644 --- a/app/SearchDisplace/SearchAndDisplaceFromFiles.php +++ b/app/SearchDisplace/SearchAndDisplaceFromFiles.php @@ -31,22 +31,27 @@ class SearchAndDisplaceFromFiles } $documentContent = $this->storage->get($this->documentFilePath); - $info = json_decode($this->storage->get($this->infoFilePath), true); + $searchersContent = json_decode($this->storage->get($this->infoFilePath), true); - $searchAndDisplace = new SearchAndDisplace($documentContent, $info); + $documentPath = $searchersContent['document_path']; + $searchers = $searchersContent['searchers']; - $originalDocumentPath = $info['document_path']; - $pathDetails = pathinfo($originalDocumentPath); + $this->storage->put($this->infoFilePath, json_encode($searchers[0]['content'])); - $resultedDocumentPath = $pathDetails['dirname'] . '/' . $pathDetails['filename'] . '-displaced.md'; + $searchAndDisplace = new SearchAndDisplace($documentContent, [ + 'searchers' => [ + [ + 'key' => $this->id, + 'replace_with' => $searchers[0]['replace_with'], + ] + ], + ]); try { $result = $searchAndDisplace->execute(); - file_put_contents($resultedDocumentPath, $result['content']); + file_put_contents($documentPath, $result['content']); } catch (\Exception $exception) { - \Illuminate\Support\Facades\Log::info('exception: ' . $exception->getMessage()); - return; } finally { $this->storage->delete($this->documentFilePath); diff --git a/app/SearchDisplace/Webhooks/HandleRequest.php b/app/SearchDisplace/Webhooks/HandleRequest.php index 59936f5..fbffdf6 100644 --- a/app/SearchDisplace/Webhooks/HandleRequest.php +++ b/app/SearchDisplace/Webhooks/HandleRequest.php @@ -16,7 +16,6 @@ class HandleRequest extends ProcessWebhookJob return true; } catch (\Exception $exception) { - return false; } } diff --git a/public/js/app.js b/public/js/app.js index 14f63db..1a49512 100644 --- a/public/js/app.js +++ b/public/js/app.js @@ -3487,6 +3487,12 @@ function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) { try function _asyncToGenerator(fn) { return function () { var self = this, args = arguments; return new Promise(function (resolve, reject) { var gen = fn.apply(self, args); function _next(value) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value); } function _throw(err) { asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err); } _next(undefined); }); }; } +function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; } + +function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); } + +function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; } + function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } } function _defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } @@ -3539,13 +3545,14 @@ var ProcessFile = /*#__PURE__*/function (_Vue) { _this.searchersFilters = []; // The list of selected filters/searchers - _this.selectedSearchers = []; //The list of expanded rows in the selected filters/searchers table + _this.selectedSearchers = {}; //The list of expanded rows in the selected filters/searchers table _this.expandedRows = []; // The list of options applied to the searchers (for the moment, only replace_with) _this.searchersOptions = {}; // Flag to determine whether or not we will show the diff highlights _this.showDiffHighlight = false; + _this.newlySelectedSearchers = []; return _this; } /** @@ -3608,6 +3615,24 @@ var ProcessFile = /*#__PURE__*/function (_Vue) { } else { this.searchersDialogVisible = !this.searchersDialogVisible; } + + if (!this.searchersDialogVisible) { + var _iterator = _createForOfIteratorHelper(this.newlySelectedSearchers), + _step; + + try { + for (_iterator.s(); !(_step = _iterator.n()).done;) { + var selectedSearcher = _step.value; + this.selectedSearchers[selectedSearcher.id] = selectedSearcher; + } + } catch (err) { + _iterator.e(err); + } finally { + _iterator.f(); + } + + this.newlySelectedSearchers = []; + } } /** * Wait for the file to be processed in ingest @@ -3672,6 +3697,11 @@ var ProcessFile = /*#__PURE__*/function (_Vue) { value: function onSelectedSearchersReorder($event) { this.selectedSearchers = $event.value; } + }, { + key: "confirmDeleteProduct", + value: function confirmDeleteProduct(searcher) { + this.$delete(this.selectedSearchers, searcher.id); + } /** * Run the searchers */ @@ -3690,7 +3720,7 @@ var ProcessFile = /*#__PURE__*/function (_Vue) { this.processing = true; this.processedFileContent = ''; searchers = []; - this.selectedSearchers.forEach(function (searcher) { + Object.values(this.selectedSearchers).forEach(function (searcher) { searchers.push({ 'key': searcher.id, 'replace_with': _this2.searchersOptions[searcher.id] || '' @@ -28432,7 +28462,7 @@ var render = function() { staticClass: "p-datatable-sm", attrs: { value: _vm.searchers, - selection: _vm.selectedSearchers, + selection: _vm.newlySelectedSearchers, dataKey: "id", selectionMode: "multiple", metaKeySelection: false, @@ -28449,7 +28479,7 @@ var render = function() { _vm.searchers = $event }, "update:selection": function($event) { - _vm.selectedSearchers = $event + _vm.newlySelectedSearchers = $event } } }, @@ -28476,13 +28506,13 @@ var render = function() { { staticClass: "p-datatable-sm", attrs: { - value: _vm.selectedSearchers, + value: Object.values(_vm.selectedSearchers), dataKey: "id", expandedRows: _vm.expandedRows }, on: { "update:value": function($event) { - _vm.selectedSearchers = $event + _vm.$set(Object, "values(selectedSearchers)", $event) }, "update:expandedRows": function($event) { _vm.expandedRows = $event @@ -28554,6 +28584,27 @@ var render = function() { _vm._v(" "), _c("Column", { attrs: { field: "name", header: "Name" } }), _vm._v(" "), + _c("Column", { + scopedSlots: _vm._u([ + { + key: "body", + fn: function(slotProps) { + return [ + _c("Button", { + staticClass: "p-button-rounded p-button-warning", + attrs: { icon: "pi pi-trash" }, + on: { + click: function($event) { + return _vm.confirmDeleteProduct(slotProps.data) + } + } + }) + ] + } + } + ]) + }), + _vm._v(" "), _c("Column", { attrs: { expander: true, headerStyle: "width: 3rem" } }) diff --git a/resources/js/components/ProcessFile/ProcessFile.ts b/resources/js/components/ProcessFile/ProcessFile.ts index 159042b..bf63f94 100644 --- a/resources/js/components/ProcessFile/ProcessFile.ts +++ b/resources/js/components/ProcessFile/ProcessFile.ts @@ -1,6 +1,6 @@ import marked from 'marked'; -import { Vue, Component, Prop, Watch } from 'vue-property-decorator'; -import { FileData } from '@/interfaces/FileData'; +import {Vue, Component, Prop, Watch} from 'vue-property-decorator'; +import {FileData} from '@/interfaces/FileData'; @Component export default class ProcessFile extends Vue { @@ -31,7 +31,7 @@ export default class ProcessFile extends Vue { private processedFileContentPreview: string = ''; - private documentDiffIndexes: { [key:string]: Array<{ start: number; end: number; }>; } = {}; + private documentDiffIndexes: { [key: string]: Array<{ start: number; end: number; }>; } = {}; // Flag to determine whether the text is processing or not private processing: boolean = false; @@ -49,7 +49,7 @@ export default class ProcessFile extends Vue { private searchersFilters: any = []; // The list of selected filters/searchers - private selectedSearchers: Array<{ id: string; name: string; }> = []; + private selectedSearchers: any = {}; //The list of expanded rows in the selected filters/searchers table private expandedRows: Array<{ id: string; name: string; }> = []; @@ -60,6 +60,8 @@ export default class ProcessFile extends Vue { // Flag to determine whether or not we will show the diff highlights private showDiffHighlight: boolean = false; + private newlySelectedSearchers: Array<{ id: string; name: string; }> = []; + /** * */ @@ -101,12 +103,19 @@ export default class ProcessFile extends Vue { * @param {string} newValue The new value for the dialog visibility */ private toggleSearchersDialog(newValue?: boolean) { - if (typeof newValue !== 'undefined') { this.searchersDialogVisible = newValue; } else { this.searchersDialogVisible = !this.searchersDialogVisible; } + + if ( ! this.searchersDialogVisible) { + for (let selectedSearcher of this.newlySelectedSearchers) { + this.selectedSearchers[selectedSearcher.id] = selectedSearcher; + } + + this.newlySelectedSearchers = []; + } } /** @@ -140,16 +149,18 @@ export default class ProcessFile extends Vue { } } - /** * * @param $event */ - private onSelectedSearchersReorder($event: any) - { + private onSelectedSearchersReorder($event: any) { this.selectedSearchers = $event.value; } + private confirmDeleteProduct(searcher: any) { + this.$delete(this.selectedSearchers, searcher.id); + } + /** * Run the searchers */ @@ -158,7 +169,8 @@ export default class ProcessFile extends Vue { this.processedFileContent = ''; let searchers: Array<{ key: string; replace_with: string; }> = []; - this.selectedSearchers.forEach((searcher) => { + + Object.values(this.selectedSearchers).forEach((searcher: any) => { searchers.push({ 'key': searcher.id, 'replace_with': this.searchersOptions[searcher.id] || '' @@ -180,7 +192,7 @@ export default class ProcessFile extends Vue { private createDiffPreview() { this.processedFileContentPreview = this.processedFileContent; - let indexes: Array<{start: number; end: number}> = []; + let indexes: Array<{ start: number; end: number }> = []; for (let searcher in this.documentDiffIndexes) { const searcherIndexes = this.documentDiffIndexes[searcher]; @@ -220,8 +232,7 @@ export default class ProcessFile extends Vue { * @param {boolean} oldValue */ @Watch('showDiffHighlight') - private onDiffHighlightChanged(newValue: boolean, oldValue: boolean): void - { + private onDiffHighlightChanged(newValue: boolean, oldValue: boolean): void { console.log('OLD: ', oldValue); console.log('NEW: ', newValue); } diff --git a/resources/js/components/ProcessFile/ProcessFile.vue b/resources/js/components/ProcessFile/ProcessFile.vue index a6028cc..8cd2ec5 100644 --- a/resources/js/components/ProcessFile/ProcessFile.vue +++ b/resources/js/components/ProcessFile/ProcessFile.vue @@ -128,18 +128,17 @@ - + + + + +