backup failed (CR 8.0.6)
Solved
Support
-
CR v8.0.6 (Ubuntu 24.04 LTS)
I've had backups fail twice in the past two weeks. Both times it's with a specific app that fails "Freescout". The app size is around 1.25 GB. VPS RAM is 8GB w/ 4GB swap. Actual RAM usage is 2.5 GB with no spikes in CPU or RAM that can be seen in the graphs. The VPS is with Hetzner. Storage is with Backblaze S3 via tarball. Never got a failed backup on my home servers. Just wanted to share.
Fail #1: 09/19 (might be 8.0.6 also as I manually updated immediately after noticing the notification)
Fail #2: 09/30 (8.0.6)2024-09-30T03:07:17.092Z box:tasks update 13887: {"percent":51,"message":"Copying with concurrency of 10"} 2024-09-30T03:07:17.327Z box:tasks update 13887: {"percent":51,"message":"Copying files from 0-1"} 2024-09-30T03:07:17.328Z box:tasks update 13887: {"percent":51,"message":"Copying (multipart) snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc"} 2024-09-30T03:07:17.569Z box:tasks update 13887: {"percent":51,"message":"Copying part 1 - /crdojdr/snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc bytes=0-1073741823"} 2024-09-30T03:07:17.574Z box:tasks update 13887: {"percent":51,"message":"Copying part 2 - /crdojdr/snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc bytes=1073741824-1195745267"} 2024-09-30T03:07:33.128Z box:tasks update 13887: {"percent":51,"message":"Copying part 2 - Etag: \"65304083cc925ecba1ad342908fb494f\""} 2024-09-30T03:12:39.805Z box:tasks update 13887: {"percent":51,"message":"Retrying (1) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-63add342099c498c 500"} 2024-09-30T03:18:24.550Z box:tasks update 13887: {"percent":51,"message":"Retrying (2) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-b02fe6d2aa843dd7 500"} 2024-09-30T03:23:56.621Z box:tasks update 13887: {"percent":51,"message":"Retrying (3) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-8898a5eb579eba3a 500"} 2024-09-30T03:29:35.379Z box:tasks update 13887: {"percent":51,"message":"Retrying (4) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-dbc747149827d9db 500"} 2024-09-30T03:35:12.554Z box:tasks update 13887: {"percent":51,"message":"Retrying (5) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-aae12e5e14612188 500"} 2024-09-30T03:41:06.567Z box:tasks update 13887: {"percent":51,"message":"Retrying (6) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-9ff927a230a81aa0 500"} 2024-09-30T03:46:44.311Z box:tasks update 13887: {"percent":51,"message":"Retrying (7) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-bd3b1654e21b3d2f 500"} 2024-09-30T03:52:15.398Z box:tasks update 13887: {"percent":51,"message":"Retrying (8) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-19cdfb9365783d08 500"} 2024-09-30T03:57:49.983Z box:tasks update 13887: {"percent":51,"message":"Retrying (9) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-3714b1429f62d0c1 500"} 2024-09-30T04:03:27.747Z box:tasks update 13887: {"percent":51,"message":"Retrying (10) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-8a7cbc35fc7ea878 500"} 2024-09-30T04:09:27.102Z box:tasks update 13887: {"percent":51,"message":"Retrying (11) multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc. Error: InternalError: incident id ecc02e391b43-ce2c87544921132e 500"} 2024-09-30T04:09:27.103Z box:tasks update 13887: {"percent":51,"message":"Aborting multipart copy of snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc"} 2024-09-30T04:09:27.407Z box:storage/s3 copy: s3 copy error when copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc: InternalError: incident id ecc02e391b43-ce2c87544921132e 2024-09-30T04:09:27.408Z box:tasks update 13887: {"percent":51,"message":"Copied 1 files with error: BoxError: Error copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc (1195745268 bytes): InternalError InternalError: incident id ecc02e391b43-ce2c87544921132e"} 2024-09-30T04:09:27.408Z box:backuptask copy: copied to 2024-09-30-030000-644/app_sub.mydomain.com_v1.11.12 errored. error: Error copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc (1195745268 bytes): InternalError InternalError: incident id ecc02e391b43-ce2c87544921132e 2024-09-30T04:09:27.414Z box:taskworker Task took 4166.815 seconds 2024-09-30T04:09:27.414Z box:tasks setCompleted - 13887: {"result":null,"error":{"stack":"BoxError: Error copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc (1195745268 bytes): InternalError InternalError: incident id ecc02e391b43-ce2c87544921132e\n at done (/home/yellowtent/box/src/storage/s3.js:333:48)\n at Response.<anonymous> (/home/yellowtent/box/src/storage/s3.js:409:71)\n at Request.<anonymous> (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:367:18)\n at Request.callListeners (/home/yellowtent/box/node_modules/aws-sdk/lib/sequential_executor.js:106:20)\n at Request.emit (/home/yellowtent/box/node_modules/aws-sdk/lib/sequential_executor.js:78:10)\n at Request.emit (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:686:14)\n at Request.transition (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:22:10)\n at AcceptorStateMachine.runTo (/home/yellowtent/box/node_modules/aws-sdk/lib/state_machine.js:14:12)\n at /home/yellowtent/box/node_modules/aws-sdk/lib/state_machine.js:26:10\n at Request.<anonymous> (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:38:9)","name":"BoxError","reason":"External Error","details":{},"message":"Error copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc (1195745268 bytes): InternalError InternalError: incident id ecc02e391b43-ce2c87544921132e"}} 2024-09-30T04:09:27.414Z box:tasks update 13887: {"percent":100,"result":null,"error":{"stack":"BoxError: Error copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc (1195745268 bytes): InternalError InternalError: incident id ecc02e391b43-ce2c87544921132e\n at done (/home/yellowtent/box/src/storage/s3.js:333:48)\n at Response.<anonymous> (/home/yellowtent/box/src/storage/s3.js:409:71)\n at Request.<anonymous> (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:367:18)\n at Request.callListeners (/home/yellowtent/box/node_modules/aws-sdk/lib/sequential_executor.js:106:20)\n at Request.emit (/home/yellowtent/box/node_modules/aws-sdk/lib/sequential_executor.js:78:10)\n at Request.emit (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:686:14)\n at Request.transition (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:22:10)\n at AcceptorStateMachine.runTo (/home/yellowtent/box/node_modules/aws-sdk/lib/state_machine.js:14:12)\n at /home/yellowtent/box/node_modules/aws-sdk/lib/state_machine.js:26:10\n at Request.<anonymous> (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:38:9)","name":"BoxError","reason":"External Error","details":{},"message":"Error copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc (1195745268 bytes): InternalError InternalError: incident id ecc02e391b43-ce2c87544921132e"}} BoxError: Error copying snapshot/app_a52611d6-c20e-4f40-a1cb-74b94e58eda6.tar.gz.enc (1195745268 bytes): InternalError InternalError: incident id ecc02e391b43-ce2c87544921132e at done (/home/yellowtent/box/src/storage/s3.js:333:48) at Response.<anonymous> (/home/yellowtent/box/src/storage/s3.js:409:71) at Request.<anonymous> (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:367:18) at Request.callListeners (/home/yellowtent/box/node_modules/aws-sdk/lib/sequential_executor.js:106:20) at Request.emit (/home/yellowtent/box/node_modules/aws-sdk/lib/sequential_executor.js:78:10) at Request.emit (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:686:14) at Request.transition (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:22:10) at AcceptorStateMachine.runTo (/home/yellowtent/box/node_modules/aws-sdk/lib/state_machine.js:14:12) at /home/yellowtent/box/node_modules/aws-sdk/lib/state_machine.js:26:10 at Request.<anonymous> (/home/yellowtent/box/node_modules/aws-sdk/lib/request.js:38:9) -
G girish marked this topic as a question on
-
I've had the same problem since the beginning of august I think and Backblaze pretends it's normal. Because of the way Backblaze is build we can expect to get errors 500 from time to time and we should just try again. As far as I know the backup script from cloudron already tries a certain number of times before giving up so I don't know if there is a solution, other than increasing further the number of attempts.
-
Any recommendations for an alternate backup provider?
-
I emailed Backblaze support about this a few days ago. Their last response is:
From what my engineers are telling me, it looks like there is an issue on our end that they are investigating. As far as I can tell from what they are saying it is not a network issue on your end. I do not have any ETA at this time on when this will be resolved but it is being looked into.
-
J joseph has marked this topic as solved on
-
Just today I got a notification about an erroneous backup of the server where I have Nextcloud installed (150GB size) and I backup once a week.. I should add that the previous backups were successful. The provider is Backblaze B2.
Oct 06 01:02:32 box:storage/s3 Upload progress: {"loaded":1073741824,"part":2,"key":"102/snapshot/app_50d07945-bf1b-4345-8e8c-c146e86c12a7.tar.gz.enc"} Jan 01 01:24:00 node:events:496 [no timestamp] throw er; // Unhandled 'error' event [no timestamp] ^ [no timestamp] [no timestamp] write EPIPE [no timestamp] at WriteWrap.onWriteComplete [as oncomplete] (node:internal/stream_base_commons:94:16) [no timestamp] 'error' event on TLSSocket instance at: [no timestamp] at emitErrorNT (node:internal/streams/destroy:169:8) [no timestamp] at emitErrorCloseNT (node:internal/streams/destroy:128:3) [no timestamp] at process.processTicksAndRejections (node:internal/process/task_queues:82:21) { [no timestamp] errno: -32, [no timestamp] code: 'EPIPE', [no timestamp] syscall: 'write' [no timestamp] } [no timestamp] [no timestamp] v20.12.2 Oct 06 01:02:32 box:shell backup-snapshot/app_50d07945-bf1b-4345-8e8c-c146e86c12a7: /usr/bin/sudo -S -E --close-from=4 /home/yellowtent/box/src/scripts/backupupload.js snapshot/app_50d07945-bf1b-4345-8e8c-c146e86c12a7 tgz {"localRoot":"/home/yellowtent/appsdata/50d07945-bf1b-4345-8e8c-c146e86c12a7","layout":[]} errored BoxError: backup-snapshot/app_50d07945-bf1b-4345-8e8c-c146e86c12a7 exited with code 1 signal null [no timestamp] at ChildProcess.<anonymous> (/home/yellowtent/box/src/shell.js:122:19) [no timestamp] at ChildProcess.emit (node:events:518:28) [no timestamp] at ChildProcess.emit (node:domain:488:12) [no timestamp] at ChildProcess._handle.onexit (node:internal/child_process:294:12) { [no timestamp] reason: 'Shell Error', [no timestamp] details: {}, [no timestamp] code: 1, [no timestamp] signal: null [no timestamp] } Oct 06 01:02:32 box:taskworker Task took 152.05 seconds Oct 06 01:02:32 box:tasks setCompleted - 2443: {"result":null,"error":{"stack":"BoxError: Backuptask crashed\n at runBackupUpload (/home/yellowtent/box/src/backuptask.js:164:15)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async uploadAppSnapshot (/home/yellowtent/box/src/backuptask.js:361:5)\n at async backupAppWithTag (/home/yellowtent/box/src/backuptask.js:383:5)\n at async fullBackup (/home/yellowtent/box/src/backuptask.js:504:29)","name":"BoxError","reason":"Internal Error","details":{},"message":"Backuptask crashed"}} Oct 06 01:02:32 box:tasks update 2443: {"percent":100,"result":null,"error":{"stack":"BoxError: Backuptask crashed\n at runBackupUpload (/home/yellowtent/box/src/backuptask.js:164:15)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async uploadAppSnapshot (/home/yellowtent/box/src/backuptask.js:361:5)\n at async backupAppWithTag (/home/yellowtent/box/src/backuptask.js:383:5)\n at async fullBackup (/home/yellowtent/box/src/backuptask.js:504:29)","name":"BoxError","reason":"Internal Error","details":{},"message":"Backuptask crashed"}} [no timestamp] Backuptask crashed [no timestamp] at runBackupUpload (/home/yellowtent/box/src/backuptask.js:164:15) [no timestamp] at process.processTicksAndRejections (node:internal/process/task_queues:95:5) [no timestamp] at async uploadAppSnapshot (/home/yellowtent/box/src/backuptask.js:361:5) [no timestamp] at async backupAppWithTag (/home/yellowtent/box/src/backuptask.js:383:5) [no timestamp] at async fullBackup (/home/yellowtent/box/src/backuptask.js:504:29)On other servers the backups are successful.
