CRON clean temp (file and DB)

src/Entity/Message/Email.php

@@ -36,6 +36,7 @@ class Email extends AbstractMessage
     private Access $author;
 
     #[ORM\OneToMany(mappedBy: 'email', targetEntity: ReportEmail::class, cascade: ['persist'], orphanRemoval: true)]
+    #[ORM\JoinColumn(onDelete: 'cascade')]
     private Collection $reports;
 
     #[ORM\ManyToOne(cascade: ['persist'])]
@@ -50,8 +51,8 @@ class Email extends AbstractMessage
 
     #[ORM\ManyToMany(targetEntity: Tagg::class, inversedBy: 'emails', cascade: ['persist'])]
     #[ORM\JoinTable(name: 'tag_message')]
-    #[ORM\JoinColumn(name: 'message_id', referencedColumnName: 'id')]
-    #[ORM\InverseJoinColumn(name: 'tag_id', referencedColumnName: 'id')]
+    #[ORM\JoinColumn(name: 'message_id', referencedColumnName: 'id', onDelete: 'cascade')]
+    #[ORM\InverseJoinColumn(name: 'tag_id', referencedColumnName: 'id', onDelete: 'cascade')]
     protected Collection $tags;
 
     public function __construct()

src/Service/Cron/Job/CleanDb.php

@@ -16,19 +16,13 @@ use JetBrains\PhpStorm\Pure;
  *
  * >>> ot:cron clean-db
  */
-// TODO: voir si ajouter :
-// * Information (282.000 lignes)? (avec NotificationUser, 169.000 lignes)
-// * MigrationLog (972.000 lignes)?
-// * ReportMessage? (4.500.000 lignes)
-
-// TODO: table potentiellement à supprimer dans la foulée : AccessTmp, Person_ori, Person_save, MigrationLog
-
 class CleanDb extends BaseCronJob implements CronjobInterface
 {
     /**
-     * Delay before deleting the record of the temporary file from the DB
+     * Delay before deleting the record from the DB
      */
-    private const PURGE_RECORDS_OLDER_THAN = 365;
+    private const PURGE_RECORDS_OLDER_THAN = 60;
+    private const PURGE_AUDIT_RECORDS_OLDER_THAN = 180;
 
     /**
      * @param Connection $connection
@@ -70,23 +64,31 @@ class CleanDb extends BaseCronJob implements CronjobInterface
         $maxDate = DatesUtils::new();
         $maxDate->sub(new \DateInterval('P' . self::PURGE_RECORDS_OLDER_THAN . 'D'));
 
+        $maxDateAudit = DatesUtils::new();
+        $maxDateAudit->sub(new \DateInterval('P' . self::PURGE_AUDIT_RECORDS_OLDER_THAN . 'D'));
+
         $this->ui->print('Purge DB from temporary records modified before ' . $maxDate->format('c'));
         $this->connection->beginTransaction();
+        $this->connection->setAutoCommit(false);
 
         $purged = 0;
 
-        $purged += $this->purgeAuditTables($maxDate);
-//        $purged += $this->purgeMessages($maxDate);
-        $purged += $this->purgeLoginLog($maxDate);
-
-        if ($commit) {
-            $this->connection->commit();
-            $this->ui->print('DB purged - ' . $purged . ' records permanently deleted');
-            return;
+        try{
+            $purged += $this->purgeAuditTables($maxDateAudit);
+            $purged += $this->purgeMessages($maxDate);
+            $purged += $this->purgeNotifications($maxDate);
+
+            if ($commit) {
+                $this->connection->commit();
+                $this->ui->print('DB purged - ' . $purged . ' records permanently deleted');
+                return;
+            }else{
+                $this->connection->rollback();
+                $this->ui->print('DB purged - ' . $purged . ' records would be permanently deleted');
+            }
+        }catch (\Throwable $exception){
+            $this->connection->rollback();
         }
-
-        $this->connection->rollback();
-        $this->ui->print('DB purged - ' . $purged . ' records would be permanently deleted');
     }
 
     /**
@@ -128,7 +130,6 @@ class CleanDb extends BaseCronJob implements CronjobInterface
 
     /**
      * Purge Message table and returns the number of deleted records
-     * TODO: à confirmer
      *
      * @param DateTime $maxDate
      * @return int
@@ -136,27 +137,37 @@ class CleanDb extends BaseCronJob implements CronjobInterface
      */
     protected function purgeMessages(DateTime $maxDate): int
     {
+        $this->ui->print('Purge DB from records of message create before ' . $maxDate->format('c'));
+
         $q = $this->connection->createQueryBuilder();
-        $q->delete('Message')->where($q->expr()->lt('dateSent', $maxDate->format('Y-m-d')));
 
-        // TODO: tester si les tables message_documents et message_files sont bien nettoyées en cascade ou s'il faut
-        //       le faire manuellement
+        $q->delete('Message')
+            ->andWhere( $q->expr()->gt('id', 0))
+            ->andWhere( $q->expr()->lt('dateSent', $maxDate->format('Y-m-d')))
+            ->andWhere( $q->expr()->eq('isSystem', true))
+        ;
+
         return $q->execute();
     }
 
     /**
-     * Purge LoginLog table and returns the number of deleted records
+     * Purge Information (Notification Discr) table and returns the number of deleted records
      *
      * @param DateTime $maxDate
      * @return int
      * @throws \Doctrine\DBAL\Exception
      */
-    protected function purgeLoginLog(DateTime $maxDate): int
+    protected function purgeNotifications(DateTime $maxDate): int
     {
+        $this->ui->print('Purge DB from records of notifications create before ' . $maxDate->format('c'));
+
         $q = $this->connection->createQueryBuilder();
-        $q->delete('LoginLog')->where($q->expr()->lt('date', $maxDate->format('Y-m-d')));
+
+        $q->delete('Information')
+            ->andWhere( $q->expr()->eq('discr', '"notification"'))
+            ->andWhere( $q->expr()->lt('createDate', $maxDate->format('Y-m-d')))
+        ;
 
         return $q->execute();
     }
-
 }

src/Service/Cron/Job/CleanTempFiles.php

@@ -2,15 +2,16 @@
 
 namespace App\Service\Cron\Job;
 
-use App\Repository\Access\AccessRepository;
+use App\Entity\Core\File;
+use App\Enum\Core\FileHostEnum;
+use App\Enum\Core\FileStatusEnum;
 use App\Repository\Core\FileRepository;
 use App\Service\Cron\BaseCronJob;
 use App\Service\Cron\CronjobInterface;
 use App\Service\File\Storage\LocalStorage;
 use App\Service\Utils\DatesUtils;
-use Doctrine\Common\Collections\Collection;
-use Doctrine\Common\Collections\Criteria;
-use Doctrine\ORM\EntityManagerInterface;
+use Doctrine\DBAL\Connection;
+use Doctrine\ORM\QueryBuilder;
 use Exception;
 use JetBrains\PhpStorm\Pure;
 
@@ -24,29 +25,17 @@ class CleanTempFiles extends BaseCronJob implements CronjobInterface
     /**
      * Delay before removing a temporary file (in days)
      */
-    private const DELETE_OLDER_THAN = 7;
-
-    /**
-     * ID of the Access who author the file deletion
-     */
-    private const AUTHOR = 10984;
-
-    /**
-     * Delay before deleting the record of the temporary file from the DB
-     */
-    private const PURGE_RECORDS_OLDER_THAN = 365;
+    private const DELETE_OLDER_THAN = 60;
 
     /**
+     * @param Connection $connection
      * @param FileRepository $fileRepository
-     * @param EntityManagerInterface $em
-     * @param AccessRepository $accessRepository
      * @param LocalStorage $storage
      */
     #[Pure]
     public function __construct(
+        private Connection $connection,
         private FileRepository $fileRepository,
-        private EntityManagerInterface $em,
-        private AccessRepository $accessRepository,
         private LocalStorage $storage
     ) {
         parent::__construct();
@@ -58,7 +47,8 @@ class CleanTempFiles extends BaseCronJob implements CronjobInterface
      */
     public function preview(): void
     {
-        $files = $this->listFilesToDelete();
+        $maxDate = DatesUtils::new();
+        $files = $this->listFilesToDelete($maxDate);
         $total = count($files);
         $this->ui->print($total . " temporary files to be removed");
         $this->ui->print("> Printing the first and last 50 :");
@@ -73,7 +63,7 @@ class CleanTempFiles extends BaseCronJob implements CronjobInterface
             $this->ui->print('  * ' . $file->getPath());
         }
 
-        $this->purgeDb(false);
+        $this->purgeDb($maxDate, false);
     }
 
     /**
@@ -82,56 +72,75 @@ class CleanTempFiles extends BaseCronJob implements CronjobInterface
      */
     public function execute(): void
     {
-        $files = $this->listFilesToDelete();
-
+        $maxDate = DatesUtils::new();
+        $files = $this->listFilesToDelete($maxDate);
         $this->deleteFiles($files);
-
-        $this->purgeDb();
+        $this->purgeDb($maxDate);
     }
 
     /**
      * List the files to delete in the DB
-     *
-     * @return Collection
+     * @param \DateTime $maxDate
+     * @return array<File>
      * @throws Exception
      */
-    protected function listFilesToDelete(): Collection
+    protected function listFilesToDelete(\DateTime $maxDate): array
     {
+        $this->ui->print('List temporary files created before ' . $maxDate->format('c'));
 
-        $maxDate = DatesUtils::new();
-        $maxDate->sub(new \DateInterval('P' . self::DELETE_OLDER_THAN . 'D'));
+        $queryBuilder = $this->fileRepository->createQueryBuilder('f');
+        $queryBuilder->select();
+        $this->getQueryConditions($queryBuilder, $maxDate);
+        return $queryBuilder->getQuery()->getResult();
+    }
 
-        $this->ui->print('List temporary files created before ' . $maxDate->format('c'));
+    /**
+     * Purge File table and returns the number of deleted records
+     *
+     * @param \DateTime $maxDate
+     * @param bool $commit
+     * @return int
+     */
+    protected function purgeDb(\DateTime $maxDate, bool $commit = true, ): void {
+        $this->ui->print('Purge DB from records of temporary files create before ' . $maxDate->format('c'));
 
-        $criteria = new Criteria();
-        $criteria->where(
-            Criteria::expr()->andX(
-                Criteria::expr()->eq('isTemporaryFile', true),
-                Criteria::expr()->orX(
-                    Criteria::expr()->lt('createDate', $maxDate),
-                    Criteria::expr()->isNull('createDate')
-                )
-            )
-        );
+        $this->connection->beginTransaction();
+        $this->connection->setAutoCommit(false);
 
-        return $this->fileRepository->matching($criteria);
+        $purged = 0;
+
+        try{
+            $purged += $this->purgeFile($maxDate);
+
+            if ($commit) {
+                $this->connection->commit();
+                $this->ui->print('DB purged - ' . $purged . ' records permanently deleted');
+            }else{
+                $this->connection->rollback();
+                $this->ui->print('DB purged - ' . $purged . ' records would be permanently deleted');
+            }
+        }catch (\Throwable $exception){
+            $this->connection->rollback();
+        }
+    }
+
+    private function purgeFile($maxDate){
+        $queryBuilder = $this->fileRepository->createQueryBuilder('f');
+        $queryBuilder->delete();
+        $this->getQueryConditions($queryBuilder, $maxDate);
+        return $queryBuilder->getQuery()->execute();
     }
 
     /**
      * Delete the files
      *
-     * @param Collection $files
+     * @param array $files
+     * @return void
      */
-    protected function deleteFiles(Collection $files): void {
+    protected function deleteFiles(array $files): void {
         $total = count($files);
         $this->ui->print($total . " temporary files to be removed");
 
-        $author = $this->accessRepository->find(self::AUTHOR);
-
-        if ($author === null) {
-            throw new \RuntimeException('Access ' . self::AUTHOR . ' could not be found');
-        }
-
         $this->ui->print('Deleting files...');
         $i = 0;
         $deleted = 0;
@@ -140,7 +149,7 @@ class CleanTempFiles extends BaseCronJob implements CronjobInterface
             try {
                 $i++;
                 $this->ui->progress($i, $total);
-                $this->storage->delete($file, $author);
+                $this->storage->forceDelete($file);
                 $deleted++;
             } catch (\RuntimeException $e) {
                 $this->ui->print('ERROR : ' . $e->getMessage());
@@ -151,38 +160,29 @@ class CleanTempFiles extends BaseCronJob implements CronjobInterface
     }
 
     /**
-     * Purge the DB from temporary file records older than N days
-     *
-     * @param bool $commit
-     * @throws Exception
+     * @param QueryBuilder $queryBuilder
+     * @param \DateTime $maxDate
+     * @return mixed
      */
-    protected function purgeDb(bool $commit = true): void {
-
-        $maxDate = DatesUtils::new();
-        $maxDate->sub(new \DateInterval('P' . self::PURGE_RECORDS_OLDER_THAN . 'D'));
-
-        $this->ui->print('Purge DB from records of files deleted before ' . $maxDate->format('c'));
-
-        $this->em->beginTransaction();
-
-        $queryBuilder = $this->em->createQueryBuilder();
-        $q = $queryBuilder
-            ->delete('File', 'f')
-            ->where($queryBuilder->expr()->eq('f.isTemporaryFile', 1))
-            ->andWhere($queryBuilder->expr()->lt('f.updateDate', ':maxDate'))
-            ->setParameter('maxDate', $maxDate)
-            ->getQuery();
-
-        $purged = $q->getResult();
-
-        if ($commit) {
-            $this->em->commit();
-            $this->ui->print('DB purged - ' . $purged . ' records permanently deleted');
-            return;
-        }
-
-        $this->em->rollback();
-        $this->ui->print('DB purged - ' . $purged . ' records would be permanently deleted');
+    private function getQueryConditions(QueryBuilder $queryBuilder, \DateTime $maxDate): void{
+        $queryBuilder
+            ->andWhere(
+                $queryBuilder->expr()->orX(
+                    $queryBuilder->expr()->eq('f.isTemporaryFile', ':temporaryTrue'),
+                    $queryBuilder->expr()->eq('f.status', ':status')
+                )
+            )
+            ->andWhere($queryBuilder->expr()->eq('f.host', ':host'))
+            ->andWhere(
+                $queryBuilder->expr()->orX(
+                    $queryBuilder->expr()->lt('f.createDate', ':maxDate'),
+                    $queryBuilder->expr()->isNull('f.createDate')
+                )
+            )
+            ->setParameter('temporaryTrue', true)
+            ->setParameter('host', FileHostEnum::AP2I()->getValue())
+            ->setParameter('status', FileStatusEnum::DELETED()->getValue())
+            ->setParameter('maxDate', $maxDate->format('Y-m-d'))
+        ;
     }
-
 }

src/Service/File/Storage/LocalStorage.php

@@ -235,12 +235,6 @@ class LocalStorage implements FileStorageInterface
      */
     public function delete(File $file, Access $author): File
     {
-        $deleted = $this->filesystem->delete($file->getSlug());
-
-        if (!$deleted) {
-            throw new RuntimeException('File `' . $file->getSlug() . '` could\'nt be deleted');
-        }
-
         $file->setStatus(FileStatusEnum::DELETED()->getValue())
              ->setSize(0)
              ->setUpdatedBy($author->getId());
@@ -248,6 +242,18 @@ class LocalStorage implements FileStorageInterface
         return $file;
     }
 
+    /**
+     * @param File $file
+     */
+    public function forceDelete(File $file): void
+    {
+        $deleted = $this->filesystem->delete($file->getSlug());
+
+        if (!$deleted) {
+            throw new RuntimeException('File `' . $file->getSlug() . '` could\'nt be deleted');
+        }
+    }
+
     /**
      * If an organization owns the file, the prefix will be '(_temp_/)organization/{id}(/{type})'.
      * If a person owns it, the prefix will be '(_temp_/)person/{id}(/{type})'