/var/lib/docker/ is filling up /dev/sda2 primary partition?

[natzilla]

@nebulon
I've done another test now

I had to reinstall cubby as the file deletion I did broke the upload function on cubby.
First thing I did was move cubby to the second partition before any upload.

Before

After

We can see it increased on the primary partition

Now I moved them over to another location off app, and the space still shows being used by those uploads.

checking the docker folder again I can see it's retaining the files I moved off app

So it seems as if this might be a bug?

[nebulon]

@natzilla I wonder what that volume is, maybe this is the /tmp volume from the app? I don't think I have implemented the cleanup of files after upload as eventually they will get removed from /tmp automatically, I have to check though what the exact behavior there is.

Can you maybe run docker ps, then find the container ID of that cubby instance and then run docker inspect <containerid> and check what that volume is used for there?

[natzilla]

@nebulon

There are two containers

and

containers

[nebulon]

@natzilla the second one is the sftp service container from the platform, that is fine, but it also does not reference that volume in question (the volume with ID starting with "8aa2b....") for that please run docker inspect <appcontainerid> and check what that volume is mounted into in the container.

[natzilla]

@nebulon

I did run that one. It's the first image in my last reply. We can see 8aa2 is listed there.

[nebulon]

@natzilla can you run this without the | grep volume as it is a json file and the lines around that line with the volume id will indicate for what it is used.

[natzilla]

@nebulon

As requested

root@pon3-server:/var/lib/docker/volumes/8aa2bbafb0b8d278a3af3d83c71c68e8643cbb22ce849050c6cd2fb3a45bf30c/_data# docker inspect bb17d9e09952
[
{
"Id": "bb17d9e0995289960d36072be40786b7efda022c2ee8af80d382d78586a38d75",
"Created": "2022-12-14T15:29:16.192977943Z",
"Path": "/app/code/start.sh",
"Args": [],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 107746,
"ExitCode": 0,
"Error": "",
"StartedAt": "2022-12-14T15:29:16.794542023Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:61e1032f9a2bbb21c35ce19efbc2b232a774c49c66216c4064a1db2b58ee1962",
"ResolvConfPath": "/var/lib/docker/containers/bb17d9e0995289960d36072be40786b7efda022c2ee8af80d382d78586a38d75/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/bb17d9e0995289960d36072be40786b7efda022c2ee8af80d382d78586a38d75/hostname",
"HostsPath": "/var/lib/docker/containers/bb17d9e0995289960d36072be40786b7efda022c2ee8af80d382d78586a38d75/hosts",
"LogPath": "",
"Name": "/aa3aad90-42e4-4086-bfc4-ff806b12e78a",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "docker-cloudron-app",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "syslog",
"Config": {
"syslog-address": "udp://127.0.0.1:2514",
"syslog-format": "rfc5424",
"tag": "aa3aad90-42e4-4086-bfc4-ff806b12e78a"
}
},
"NetworkMode": "cloudron",
"PortBindings": {},
"RestartPolicy": {
"Name": "unless-stopped",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": [],
"CapDrop": [
"NET_RAW"
],
"CgroupnsMode": "host",
"Dns": [
"172.18.0.1"
],
"DnsOptions": null,
"DnsSearch": [
"."
],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": true,
"SecurityOpt": [
"apparmor=docker-cloudron-app"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 512,
"Memory": 483393536,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": null,
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 536870912,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"Mounts": [
{
"Type": "bind",
"Source": "/mnt/volumes/d828f00dc01d4eb5be43ca61defeb4b5/dropbox",
"Target": "/app/data"
}
],
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/9c64364d1e56b61ec0b8054ae1264890152caed47165466e9af88b84f392d2d8-init/diff:/var/lib/docker/overlay2/2239409d9d6dd34f365e40a3f8cee52897db3b0521d0b6a6a16b3f2726dca4c3/diff:/var/lib/docker/overlay2/9d41c30e6e6bc5e47c3c02c8b49fe9b31d95fe928af3c735235caca8f85a28e6/diff:/var/lib/docker/overlay2/c4853771022610997fe96328d237d3d7fd8e05e958153b3415abe5df7de3f076/diff:/var/lib/docker/overlay2/240c4bcf910cbbc49698a04b95da6a17257807cb252f36f618358b1ae042c0c4/diff:/var/lib/docker/overlay2/ae2ec42c74ca7ff1c2121271c438a06d5f43517a021fa27fedb3de5adfd4126e/diff:/var/lib/docker/overlay2/74f204a2f887a42c80528e7dd486b39a9e7e7f18c65dbfd45f6b0528d4209434/diff:/var/lib/docker/overlay2/40eb436639d648361086d6b347f29c52513893720131ea3c357ba186aab10c53/diff:/var/lib/docker/overlay2/419d743c3a5cd0ab05182f78265564c4d04eb1fec8339ac710d53e3769794d83/diff:/var/lib/docker/overlay2/ff1cfbb2ae944f740122a19ef2dab2d43373c440eb3c7169d2943e1851b1f2b2/diff:/var/lib/docker/overlay2/8dc58f88f40e247dd3d93507fb311e96012157cb01b8e1fe9750d7708dbfc2e6/diff:/var/lib/docker/overlay2/b122bc434a62507a71aebae852e442132b8670ed9671f4a05f45ab9d07b13278/diff:/var/lib/docker/overlay2/c73304048cfc522dc8b1d46cda84e34401cb583e1f808227754bd6c5a557e6bf/diff:/var/lib/docker/overlay2/154763b48008e44bd3de952b70756cc9e2ee9c28bfad1e6b6680ff7b11f170af/diff:/var/lib/docker/overlay2/ab54942b6e3a8214edb7740c2e40ad9a7108df32900d610272901c2c51fc72b0/diff:/var/lib/docker/overlay2/cc5d3a9f9e407cfa105df870d36b4b46a07d45223f53b5cc7a4e3f7cb8348307/diff:/var/lib/docker/overlay2/c426c24563d905ce79e17964881fae9aec29e3bd60db8e1d0c4f915c916de735/diff:/var/lib/docker/overlay2/80fb267c7479664dac1d9f28c77050536144f50357fc1996dbe7cd35d438ff53/diff:/var/lib/docker/overlay2/5a25242ee7066afe2461f8be8f607d2c7d78d0ec14ee4d84655a805987c256b4/diff:/var/lib/docker/overlay2/f557a9ff958a680b079ec20ad3adcdd1e5760d20b6570474ca7556239b2d590e/diff:/var/lib/docker/overlay2/a49d9997d75315f730f1aa7d35b0bd550fd85cafd704ddb8a24f7d4deb0f7b9e/diff",
"MergedDir": "/var/lib/docker/overlay2/9c64364d1e56b61ec0b8054ae1264890152caed47165466e9af88b84f392d2d8/merged",
"UpperDir": "/var/lib/docker/overlay2/9c64364d1e56b61ec0b8054ae1264890152caed47165466e9af88b84f392d2d8/diff",
"WorkDir": "/var/lib/docker/overlay2/9c64364d1e56b61ec0b8054ae1264890152caed47165466e9af88b84f392d2d8/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "a9d5b4ebb3669ff2adc9048e3317657d9db8da14a1e86bab850a1c40464a0b57",
"Source": "/var/lib/docker/volumes/a9d5b4ebb3669ff2adc9048e3317657d9db8da14a1e86bab850a1c40464a0b57/_data",
"Destination": "/run",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/mnt/volumes/d828f00dc01d4eb5be43ca61defeb4b5/dropbox",
"Destination": "/app/data",
"Mode": "",
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "volume",
"Name": "9120108234b71ccd3f3a7eed56ab718e0145018a80a7a6982fc9904cf25c2792",
"Source": "/var/lib/docker/volumes/9120108234b71ccd3f3a7eed56ab718e0145018a80a7a6982fc9904cf25c2792/_data",
"Destination": "/home/cloudron/.cache",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "volume",
"Name": "db9dcbde32d2b270f24b03c7e6820fe86ba4b6440275f2ccf623931faa992ccd",
"Source": "/var/lib/docker/volumes/db9dcbde32d2b270f24b03c7e6820fe86ba4b6440275f2ccf623931faa992ccd/_data",
"Destination": "/root/.cache",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "volume",
"Name": "8aa2bbafb0b8d278a3af3d83c71c68e8643cbb22ce849050c6cd2fb3a45bf30c",
"Source": "/var/lib/docker/volumes/8aa2bbafb0b8d278a3af3d83c71c68e8643cbb22ce849050c6cd2fb3a45bf30c/_data",
"Destination": "/tmp",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "aa3aad90-42e4-4086-bfc4-ff806b12e78a",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": true,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"CLOUDRON=1",
"CLOUDRON_PROXY_IP=172.18.0.1",
"CLOUDRON_APP_HOSTNAME=aa3aad90-42e4-4086-bfc4-ff806b12e78a",
"CLOUDRON_WEBADMIN_ORIGIN=----------------------.net",
"CLOUDRON_API_ORIGIN=--------------------------.net",
"CLOUDRON_APP_ORIGIN=-------------------.net",
"CLOUDRON_APP_DOMAIN=---------------------.net",
"CLOUDRON_POSTGRESQL_URL=postgres://useraa3aad9042e44086bfc4ff806b12e78a:cda2df1e66dbfeb5b419e08176cb94d5927f7cceda16dae0b5d1e140be9690e6f5bc9617c5613487c0990b2f9cefadf60bd160131454a3c2336251c1c32b109c@postgresql/dbaa3aad9042e44086bfc4ff806b12e78a",
"CLOUDRON_POSTGRESQL_USERNAME=useraa3aad9042e44086bfc4ff806b12e78a",
"CLOUDRON_POSTGRESQL_PASSWORD=cda2df1e66dbfeb5b419e08176cb94d5927f7cceda16dae0b5d1e140be9690e6f5bc9617c5613487c0990b2f9cefadf60bd160131454a3c2336251c1c32b109c",
"CLOUDRON_POSTGRESQL_HOST=postgresql",
"CLOUDRON_POSTGRESQL_PORT=5432",
"CLOUDRON_POSTGRESQL_DATABASE=dbaa3aad9042e44086bfc4ff806b12e78a",
"CLOUDRON_LDAP_SERVER=172.18.0.1",
"CLOUDRON_LDAP_HOST=172.18.0.1",
"CLOUDRON_LDAP_PORT=3002",
"CLOUDRON_LDAP_URL=ldap://172.18.0.1:3002",
"CLOUDRON_LDAP_USERS_BASE_DN=ou=users,dc=cloudron",
"CLOUDRON_LDAP_GROUPS_BASE_DN=ou=groups,dc=cloudron",
"CLOUDRON_LDAP_BIND_DN=cn=aa3aad90-42e4-4086-bfc4-ff806b12e78a,ou=apps,dc=cloudron",
"CLOUDRON_LDAP_BIND_PASSWORD=aff4d7cf97262f6b6302803b873295880a45c3aa8f33cc8bde3efff901578391273ffb4f11168342659a0ae0dfac4ed9e35e1ef70f64f23a53a3d5d030481c96",
"PATH=/usr/local/node-16.13.1/bin:/usr/local/go-1.17.5/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"DEBIAN_FRONTEND=noninteractive",
"GOROOT=/usr/local/go-1.17.5",
"CONTAINER_NAME=postgres-server-cubby",
"PGPASSWORD=",
"DEBUG=cubby*"
],
"Cmd": [
"/app/code/start.sh"
],
"Image": "cloudron/io.cloudron.cubby:20221023-213915-3426f3b61",
"Volumes": {
"/home/cloudron/.cache": {},
"/root/.cache": {},
"/run": {},
"/tmp": {}
},
"WorkingDir": "/app/code",
"Entrypoint": null,
"OnBuild": null,
"Labels": {
"appId": "aa3aad90-42e4-4086-bfc4-ff806b12e78a",
"fqdn": "----------------------.net",
"isCloudronManaged": "true",
"isSubcontainer": "false"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "f776f5c7246279bd183a4721e1d7db762fa76ac3a9bf7fd21d255c9c9a56c659",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": "/var/run/docker/netns/f776f5c72462",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"cloudron": {
"IPAMConfig": {
"IPv4Address": "172.18.16.166"
},
"Links": null,
"Aliases": [
"aa3aad90-42e4-4086-bfc4-ff806b12e78a",
"bb17d9e09952"
],
"NetworkID": "805d05f75bbd29d71f2e6dbf9b34dd5d4fdfe81f2762742b7b576b2ac80b9cc8",
"EndpointID": "5ea9bcd5c560cc945f4c4f21067a21d8ee3168c7c54d93daf94322e71eae75c7",
"Gateway": "172.18.0.1",
"IPAddress": "172.18.16.166",
"IPPrefixLen": 16,
"IPv6Gateway": "fd00:c107:d509::1",
"GlobalIPv6Address": "fd00:c107:d509::9",
"GlobalIPv6PrefixLen": 64,
"MacAddress": "02:42:ac:12:10:a6",
"DriverOpts": null
}
}
}
}
]
root@pon3-server:/var/lib/docker/volumes/8aa2bbafb0b8d278a3af3d83c71c68e8643cbb22ce849050c6cd2fb3a45bf30c/_data#

[nebulon]

ah thanks, so it is the /tmp folder. I think it is related to the upload cache in cubby. Will try to reproduce this tomorrow and also hopefully fix it in cubby then.

[natzilla]

@nebulon

Awesome, I appreciate it.

[nebulon]

Its quite busy right now here, so until the app package is fixed, it should be safe to use the webterminal and remove those files from the /tmp folder within the app to clear out some space in the meantime.

[natzilla]

@nebulon

Acknowledged, I will do so until it's patched.

[nebulon]

Fixed with cubby 1.1.0

[natzilla]

@nebulon

Awesome, thanks again.