Delete of multiple release bundles when using Derby DB may fail with DB deadlock.

2019-09-19 09:26:43,931 [http-nio-8081-exec-17] [ERROR] (o.a.r.c.e.m.GlobalExceptionMapper:48) - null
java.lang.reflect.UndeclaredThrowableException: null
	at com.sun.proxy.$Proxy94.next(Unknown Source)
	at org.artifactory.storage.db.bundle.dao.ArtifactBundlesDao.getBundlesRelatedToRepoPath(ArtifactBundlesDao.java:331)
	at org.artifactory.release.bundle.ReleaseBundleServiceImpl.getBundlesRelatedToRepoPath(ReleaseBundleServiceImpl.java:175)
	at jdk.internal.reflect.GeneratedMethodAccessor331.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:206)
	at com.sun.proxy.$Proxy253.getBundlesRelatedToRepoPath(Unknown Source)
	at org.artifactory.addon.release.bundle.interceptor.ReleaseBundleInterceptor.assertPathNotRelatedToBundle(ReleaseBundleInterceptor.java:45)
	at org.artifactory.addon.release.bundle.interceptor.ReleaseBundleInterceptor.assertDeletePathAllowed(ReleaseBundleInterceptor.java:35)
	at org.artifactory.repo.interceptor.storage.StorageInterceptorsImpl.assertDeletePathAllowed(StorageInterceptorsImpl.java:81)
	at jdk.internal.reflect.GeneratedMethodAccessor330.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:206)
	at com.sun.proxy.$Proxy172.assertDeletePathAllowed(Unknown Source)
	at org.artifactory.repo.service.RepositoryServiceImpl.fireAssertDeletePathAllowedEvent(RepositoryServiceImpl.java:2201)
	at org.artifactory.repo.service.RepositoryServiceImpl.assertDelete(RepositoryServiceImpl.java:2196)
	at org.artifactory.repo.service.RepositoryServiceImpl.undeploy(RepositoryServiceImpl.java:1301)
	at org.artifactory.repo.service.RepositoryServiceImpl.undeploy(RepositoryServiceImpl.java:1286)
	at org.artifactory.repo.service.RepositoryServiceImpl.undeploy(RepositoryServiceImpl.java:1281)
	at org.artifactory.repo.service.RepositoryServiceImpl.undeploy(RepositoryServiceImpl.java:1402)
	at jdk.internal.reflect.GeneratedMethodAccessor442.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:197)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
	at org.artifactory.request.aop.RequestAdvice.invoke(RequestAdvice.java:67)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:185)
	at org.artifactory.storage.fs.lock.aop.LockingAdvice.invoke(LockingAdvice.java:78)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:185)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212)
	at com.sun.proxy.$Proxy159.undeploy(Unknown Source)
	at org.artifactory.release.bundle.InternalReleaseBundleServiceImpl.deleteBundleContent(InternalReleaseBundleServiceImpl.java:80)
	at org.artifactory.release.bundle.InternalReleaseBundleServiceImpl.deleteBundleInternal(InternalReleaseBundleServiceImpl.java:48)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:197)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
	at org.artifactory.storage.fs.lock.aop.LockingAdvice.invoke(LockingAdvice.java:78)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:185)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212)
	at com.sun.proxy.$Proxy252.deleteBundleInternal(Unknown Source)
	at org.artifactory.release.bundle.ReleaseBundleServiceImpl.deleteBundle(ReleaseBundleServiceImpl.java:138)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:206)
	at com.sun.proxy.$Proxy253.deleteBundle(Unknown Source)
	at org.artifactory.addon.release.bundle.ReleaseBundleAddonImpl.deleteReleaseBundle(ReleaseBundleAddonImpl.java:889)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:343)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:197)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
	at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:294)
	at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:98)
	at org.artifactory.storage.fs.lock.aop.LockingAdvice.invoke(LockingAdvice.java:76)
	at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:185)
	at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212)
	at com.sun.proxy.$Proxy321.deleteReleaseBundle(Unknown Source)
	at org.artifactory.rest.resource.release.ReleaseBundleResource.deleteBundle(ReleaseBundleResource.java:193)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:76)
	at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:148)
	at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:191)
	at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:200)
	at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:103)
	at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:493)
	at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:415)
	at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:104)
	at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:277)
	at org.glassfish.jersey.internal.Errors$1.call(Errors.java:272)
	at org.glassfish.jersey.internal.Errors$1.call(Errors.java:268)
	at org.glassfish.jersey.internal.Errors.process(Errors.java:316)
	at org.glassfish.jersey.internal.Errors.process(Errors.java:298)
	at org.glassfish.jersey.internal.Errors.process(Errors.java:268)
	at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:289)
	at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:256)
	at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:703)
	at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:416)
	at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:370)
	at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:389)
	at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:342)
	at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:229)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.artifactory.webapp.servlet.RepoFilter.execute(RepoFilter.java:191)
	at org.artifactory.webapp.servlet.RepoFilter.doFilter(RepoFilter.java:97)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.artifactory.webapp.servlet.authentication.ArtifactoryAuthenticationFilterChain.lambda$doFilter$1(ArtifactoryAuthenticationFilterChain.java:135)
	at org.artifactory.webapp.servlet.authentication.PropsAuthenticationFilter.doFilter(PropsAuthenticationFilter.java:127)
	at org.artifactory.webapp.servlet.authentication.ArtifactoryAuthenticationFilterChain.doFilter(ArtifactoryAuthenticationFilterChain.java:171)
	at org.artifactory.webapp.servlet.AccessFilter.authenticateAndExecute(AccessFilter.java:311)
	at org.artifactory.webapp.servlet.AccessFilter.doFilterInternal(AccessFilter.java:208)
	at org.artifactory.webapp.servlet.AccessFilter.doFilter(AccessFilter.java:167)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.artifactory.webapp.servlet.RequestFilter.doFilter(RequestFilter.java:77)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.artifactory.webapp.servlet.ArtifactoryCsrfFilter.doFilter(ArtifactoryCsrfFilter.java:85)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.session.web.http.SessionRepositoryFilter.doFilterInternal(SessionRepositoryFilter.java:164)
	at org.springframework.session.web.http.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:80)
	at org.artifactory.webapp.servlet.SessionFilter.doFilter(SessionFilter.java:62)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.artifactory.webapp.servlet.ArtifactoryFilter.doFilter(ArtifactoryFilter.java:124)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:493)
	at org.apache.catalina.valves.rewrite.RewriteValve.invoke(RewriteValve.java:279)
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:800)
	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:800)
	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1471)
	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
	at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor68.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.jfrog.storage.wrapper.ResultSetWrapper.invoke(ResultSetWrapper.java:77)
	... 143 common frames omitted
Caused by: java.sql.SQLTransactionRollbackException: A lock could not be obtained due to a deadlock, cycle of locks and waiters is:
Lock : ROW, BUNDLE_FILES, (58,158)
  Waiting XID : {177105, S} , APP, SELECT id, name, version from artifact_bundles where id IN (SELECT DISTINCT bundle_id FROM bundle_files WHERE repo_path LIKE ?)
  Granted XID : {177099, X}
Lock : ROW, BUNDLE_FILES, (50,104)
  Waiting XID : {177099, S} , APP, SELECT id, name, version from artifact_bundles where id IN (SELECT DISTINCT bundle_id FROM bundle_files WHERE repo_path LIKE ?)
  Granted XID : {177105, X}
. The selected victim is XID : 177105.
	at org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(Unknown Source)
	at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Unknown Source)
	at org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(Unknown Source)
	at org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(Unknown Source)
	at org.apache.derby.impl.jdbc.EmbedConnection.handleException(Unknown Source)
	at org.apache.derby.impl.jdbc.ConnectionChild.handleException(Unknown Source)
	at org.apache.derby.impl.jdbc.EmbedResultSet.closeOnTransactionError(Unknown Source)
	at org.apache.derby.impl.jdbc.EmbedResultSet.movePosition(Unknown Source)
	at org.apache.derby.impl.jdbc.EmbedResultSet.next(Unknown Source)
	... 147 common frames omitted
Caused by: org.apache.derby.iapi.error.StandardException: A lock could not be obtained due to a deadlock, cycle of locks and waiters is:
Lock : ROW, BUNDLE_FILES, (58,158)
  Waiting XID : {177105, S} , APP, SELECT id, name, version from artifact_bundles where id IN (SELECT DISTINCT bundle_id FROM bundle_files WHERE repo_path LIKE ?)
  Granted XID : {177099, X}
Lock : ROW, BUNDLE_FILES, (50,104)
  Waiting XID : {177099, S} , APP, SELECT id, name, version from artifact_bundles where id IN (SELECT DISTINCT bundle_id FROM bundle_files WHERE repo_path LIKE ?)
  Granted XID : {177105, X}
. The selected victim is XID : 177105.
	at org.apache.derby.iapi.error.StandardException.newException(Unknown Source)
	at org.apache.derby.iapi.error.StandardException.newException(Unknown Source)
	at org.apache.derby.impl.services.locks.Deadlock.buildException(Unknown Source)
	at org.apache.derby.impl.services.locks.ConcurrentLockSet.lockObject(Unknown Source)
	at org.apache.derby.impl.services.locks.AbstractPool.lockObject(Unknown Source)
	at org.apache.derby.impl.services.locks.ConcurrentPool.lockObject(Unknown Source)
	at org.apache.derby.impl.store.raw.xact.RowLocking2.lockRecordForRead(Unknown Source)
	at org.apache.derby.impl.store.access.heap.HeapController.lockRow(Unknown Source)
	at org.apache.derby.impl.store.access.heap.HeapController.lockRow(Unknown Source)
	at org.apache.derby.impl.store.access.btree.index.B2IRowLocking3.lockRowOnPage(Unknown Source)
	at org.apache.derby.impl.store.access.btree.index.B2IRowLocking3._lockScanRow(Unknown Source)
	at org.apache.derby.impl.store.access.btree.index.B2IRowLockingRR.lockScanRow(Unknown Source)
	at org.apache.derby.impl.store.access.btree.BTreeForwardScan.fetchRows(Unknown Source)
	at org.apache.derby.impl.store.access.btree.BTreeScan.fetchNext(Unknown Source)
	at org.apache.derby.impl.sql.execute.TableScanResultSet.loopControl(Unknown Source)
	at org.apache.derby.impl.sql.execute.TableScanResultSet.getNextRowCore(Unknown Source)
	at org.apache.derby.impl.sql.execute.IndexRowToBaseRowResultSet.getNextRowCore(Unknown Source)
	at org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.getNextRowCore(Unknown Source)
	at org.apache.derby.impl.sql.execute.NestedLoopJoinResultSet.getNextRowCore(Unknown Source)
	at org.apache.derby.impl.sql.execute.ProjectRestrictResultSet.getNextRowCore(Unknown Source)
	at org.apache.derby.impl.sql.execute.BasicNoPutResultSetImpl.getNextRow(Unknown Source)
	... 149 common frames omitted