storage = Storage::disk('local');
$this->path = $path;
$this->type = $type;
}
/**
* @return mixed
* @throws \Exception
*/
public function execute()
{
if ($this->type === 'txt') {
return $this->path;
}
if ($this->type === 'pdf') {
// $this->convertPdfToText();
$this->convertPdfToMD();
// $this->getHtmlContentsFromPdfWithImages();
return $this->path;
}
if ($this->type !== 'docx') {
$this->convertToDocx();
}
$this->convertDocumentToText();
//$this->convertToHtml();
return $this->path;
}
/**
* Convert doc,dot,rtf,odt,pdf,docx to docx
*
*
* @return string|void
*/
private function convertToDocx()
{
(new Process(['export HOME=' . env('USER_HOME_PATH')]))->run();
/**
* Convert doc,dot,rtf,odt to docx
*/
$process = new Process([
'soffice',
'--headless',
'--convert-to',
'docx',
$this->storage->path($this->path),
'--outdir',
$this->storage->path('contracts')
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace(".$this->type", '.docx', $this->path);
}
/**
* Convert docx file to text
*
* @return void
*/
private function convertDocumentToText()
{
(new Process(['export HOME=' . env('USER_HOME_PATH')]))->run();
$process = new Process([
'soffice',
'--headless',
'--convert-to',
'txt',
$this->storage->path($this->path),
'--outdir',
$this->storage->path('contracts')
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace(['.docx', '.bin'], '.txt', $this->path);
}
protected function convertPdfToText()
{
$this->prepareForConvertPDF();
$images = $this->getImagesFromPDF();
$contents = $this->getTextContentsFromPDF();
if (!$contents && count($images) === 0) {
throw new \Exception('Could not read from file.');
}
// Handle images and image contents.
if (count($images) > 0) {
foreach ($images as $image) {
try {
$ocr = new OCR($this->storage->path($image));
$imageContents = $ocr->execute();
$contents = $contents . "\n" . $imageContents;
} catch (\Exception $exception) {
\Illuminate\Support\Facades\Log::info('something wrong: ' . $exception->getMessage());
}
}
$dir = str_replace('.pdf', '', $this->path);
$this->storage->deleteDirectory($dir);
}
$this->storage->delete($this->path);
$this->path = str_replace('.pdf', '.txt', $this->path);
$this->storage->put($this->path, $contents);
}
protected function convertPdfToMD()
{
// $this->prepareForConvertPDF();
$result = $this->getContentsFromPdf();
if ( ! $result['has_images'] && ! $result['has_text']) {
throw new \Exception('Cannot get pdf file contents.');
}
if ($result['has_text']) {
if ($result['has_images']) {
// Both text and images.
throw new \Exception('Not supported for now.');
}
// Delete directory because the contents are in the '$result' variable.
$this->storage->deleteDirectory($this->path);
$mdContents = '';
foreach ($result['htmls'] as $html) {
$converter = new HtmlConverter();
$converter->getConfig()->setOption('strip_tags', true);
$contents = $converter->convert($html);
$mdContents = $mdContents . $contents;
}
$this->path = "$this->path.md";
$this->storage->put($this->path, $mdContents);
return;
}
// Only contains images.
$imagesContent = '';
$files = $this->storage->allFiles($this->path);
foreach ($files as $file) {
// Only get the image files from the directory, it may contain some empty html files too.
if (in_array(pathinfo($file, PATHINFO_EXTENSION), ['jpg', 'png'])) {
$ocr = new OCR($this->storage->path($file));
$imagesContent = $imagesContent . $ocr->execute();
}
}
\Illuminate\Support\Facades\Log::info('============================');
\Illuminate\Support\Facades\Log::info($this->path);
// We are done with the images processing, delete directory.
$this->storage->deleteDirectory($this->path);
$this->path = "$this->path.md";
\Illuminate\Support\Facades\Log::info($this->path);
\Illuminate\Support\Facades\Log::info('++++++++++++++++++++++++++');
$this->storage->put($this->path, $imagesContent);
}
private function convertToHtml()
{
(new Process(['export HOME=' . env('USER_HOME_PATH')]))->run();
$process = new Process([
'soffice',
'--headless',
'--convert-to',
'html:HTML:EmbedImages',
$this->storage->path($this->path),
'--outdir',
$this->storage->path('contracts')
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace(".$this->type", '.html', $this->path);
}
private function convertToXML()
{
//Convert the file to xml using pdftohtml to xml and run a python scrypt to fix the paragraphs
$process = new Process([
'pdftohtml',
'-xml',
'-i',
$this->storage->path($this->path)
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = str_replace(".$this->type", '.xml', $this->path);
}
protected function prepareForConvertPDF()
{
(new Process(['export HOME=' . env('USER_HOME_PATH')]))->run();
$process = new Process([
'pip3',
'install',
'pdftotext',
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
}
protected function getImagesFromPDF()
{
$dir = str_replace('.pdf', '', $this->path);
$this->storage->makeDirectory($dir);
$process = new Process([
'pdfimages',
'-p',
$this->storage->path($this->path),
'-tiff',
$this->storage->path("$dir/ocr")
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
return $this->storage->allFiles($dir);
}
protected function getTextContentsFromPDF()
{
$outputPath = $this->storage->path(str_replace('.pdf', '.txt', $this->path));
$process = new Process([
'python3',
storage_path('scripts' . DIRECTORY_SEPARATOR . 'parse-pdf.py'),
'-i',
$this->storage->path($this->path),
'-o',
$outputPath
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
return file_get_contents($outputPath);
}
protected function getHtmlContentsFromPdfWithImages()
{
$dirName = str_replace('.pdf', '', $this->path);
$this->storage->makeDirectory($dirName);
$outputPath = $this->storage->path("$dirName/html");
$process = new Process([
'pdftohtml',
'-noframes',
$this->storage->path($this->path),
$outputPath
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = $dirName;
$converter = new HtmlConverter();
$converter->getConfig()->setOption('strip_tags', true);
$files = $this->storage->allFiles($this->path);
$htmlFileIndex = null;
foreach ($files as $index => $file) {
// if (pathinfo($file, PATHINFO_BASENAME) === 'html-html.html') {
// if (pathinfo($file, PATHINFO_EXTENSION) === 'html') {
if (pathinfo($file, PATHINFO_BASENAME) === 'html.html') {
$htmlFileIndex = $index;
break;
}
}
$htmlContents = $this->storage->get($files[$htmlFileIndex]);
$contents = $converter->convert($htmlContents);
// $this->storage->deleteDirectory($this->path);
$this->path = "$this->path.md";
$this->storage->put($this->path, $contents);
dd(3);
}
protected function getContentsFromPdf()
{
$dirName = str_replace('.pdf', '', $this->path);
$this->storage->makeDirectory($dirName);
$outputPath = $this->storage->path("$dirName/html");
$process = new Process([
'pdftohtml',
'-xml',
$this->storage->path($this->path),
$outputPath
]);
$process->run();
if (!$process->isSuccessful()) {
throw new ProcessFailedException($process);
}
$this->storage->delete($this->path);
$this->path = $dirName;
$contents = $this->storage->get("$this->path/html.xml");
$xml = simplexml_load_string($contents);
$fonts = [];
foreach ($xml->page as $page) {
foreach ($page as $p) {
if ($p->getName() === 'fontspec') {
$fonts[(int) $p['id']]['family'] = (string) $p['family'];
$fonts[(int) $p['id']]['size'] = (string) $p['size'];
$fonts[(int) $p['id']]['color'] = (string) $p['color'];
}
}
}
$htmls = [];
$hasImages = false;
$hasText = false;
try {
foreach ($xml->page as $page) {
$html = '';
$previousP = null;
foreach ($page as $p) {
if ($p->getName() == 'image') {
$html = $html . '';
$hasImages = true;
}
if ($p->getName() == 'text') {
$id = (int) $p['font'];
$font_size = $fonts[$id]['size'];
$font_color = $fonts[$id]['color'];
$font_family = $fonts[$id]['family'];
$style = '';
$style = $style . 'position: absolute;';
$style = $style . "color: $font_color;";
$style = $style . "font-family: $font_family;";
$style = $style . "font-weight: 900;";
$style = $style . "width: " . $p['width'] . "px;";
$style = $style . "height: " . $p['height'] . "px;";
$style = $style . "top: " . $p['top'] . "px;";
$style = $style . "left: " . $p['left'] . "px;";
// $style = $style . "font-size: $font_size" . "px;";
if ($p->i) {
$content = '' . $p->i . '';
} else if ($p->b) {
$content = '' . $p->b . '';
} else {
$content = $p;
}
// @TODO Must chain paragraphs if top are almost same.
$tag = $this->getTag($p, $previousP, $font_size);
$html = $html . '<' . $tag . ' style="' . $style . '">' . $content . '' . $tag . '>';
$hasText = true;
}
$previousP = $p;
}
$htmls[] = '