diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index a4046a08..ebe63459 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -391,6 +391,7 @@
       Build a puppet tarball but do not upload it anywhere.
     pre-run: playbooks/puppet-tarball/pre.yaml
     run: playbooks/puppet-tarball/run.yaml
+    files: ^metadata.json$
 
 - job:
     name: publish-openstack-puppet-branch-tarball