diff --git a/src/xterm.c b/src/xterm.c
index 54b00455a1c..18d68e52f12 100644
--- a/src/xterm.c
+++ b/src/xterm.c
@@ -11513,6 +11513,8 @@ handle_one_xevent (struct x_display_info *dpyinfo,
 #endif
 		  if (found_valuator)
 		    {
+		      x_display_set_last_user_time (dpyinfo, xev->time);
+
 		      if (fabs (total_x) > 0 || fabs (total_y) > 0)
 			{
 			  inev.ie.kind = (fabs (total_y) >= fabs (total_x)