diff --git a/plugin.sh b/plugin.sh
index f766efd..3141f47 100755
--- a/plugin.sh
+++ b/plugin.sh
@@ -75,7 +75,8 @@ run_cleanup() {
   # See https://github.com/moby/moby/issues/30942#issuecomment-540699206
   until [ -z "$(docker stack ps "$PLUGIN_STACK" -q)" ]; do sleep 1; done
   docker -H "$DOCKER_HOST" system prune --all --volumes --force
-  docker -H "$DOCKER_HOST" secret rm "$(docker -H "$DOCKER_HOST" secret ls -q)" || true
+  # try and remove all secrets; Docker will leave ones which are in use
+  docker -H "$DOCKER_HOST" secret ls --format '{{ .Name }}' | xargs -i sh -c "echo {}; docker secret -H \"$DOCKER_HOST\" rm {} || true"
   echo "--- end cleanup ---"
 }