diff --git a/src/m_menu.c b/src/m_menu.c
index 7b440d46..a0712381 100644
--- a/src/m_menu.c
+++ b/src/m_menu.c
@@ -6792,15 +6792,6 @@ void M_SetupJoystickMenu(INT32 choice)
 
 	strcpy(joystickInfo[i], "None");
 
-	// Hotplugging breaks if this block is run
-	// Because the cvar is set to 0, which disables controllers for that player
-#if 0 // #ifdef JOYSTICK_HOTPLUG
-	if (0 == cv_usejoystick.value)
-		CV_SetValue(&cv_usejoystick, 0);
-	if (0 == cv_usejoystick2.value)
-		CV_SetValue(&cv_usejoystick2, 0);
-#endif
-
 	for (i = 1; i < 8; i++)
 	{
 		if (i <= n && (I_GetJoyName(i)) != NULL)
@@ -6845,52 +6836,66 @@ static void M_AssignJoystick(INT32 choice)
 {
 #ifdef JOYSTICK_HOTPLUG
 	INT32 oldchoice;
-
-	if (choice > I_NumJoys())
-		return;
+	INT32 numjoys = I_NumJoys();
 
 	if (setupcontrols_secondaryplayer)
 	{
-		oldchoice = atoi(cv_usejoystick2.string) > I_NumJoys() ? atoi(cv_usejoystick2.string) : cv_usejoystick2.value;
+		oldchoice = atoi(cv_usejoystick2.string) > numjoys ? atoi(cv_usejoystick2.string) : cv_usejoystick2.value;
 		CV_SetValue(&cv_usejoystick2, choice);
 
 		// Just in case last-minute changes were made to cv_usejoystick.value,
 		// update the string too
-		CV_SetValue(&cv_usejoystick2, cv_usejoystick2.value);
-
-		if (oldchoice != choice)
+		// But don't do this if we're intentionally setting higher than numjoys
+		if (choice <= numjoys)
 		{
-			if (choice && oldchoice > I_NumJoys()) // if we did not select "None", we likely selected a used device
-				CV_SetValue(&cv_usejoystick2, oldchoice);
+			CV_SetValue(&cv_usejoystick2, cv_usejoystick2.value);
 
-			if (oldchoice ==
-				(atoi(cv_usejoystick2.string) > I_NumJoys() ? atoi(cv_usejoystick2.string) : cv_usejoystick2.value))
-				M_StartMessage("This joystick is used by another\n"
-				               "player. Reset the joystick\n"
-							   "for that player first.\n\n"
-							   "(Press a key)\n", NULL, MM_NOTHING);
+			// reset this so the comparison is valid
+			if (oldchoice > numjoys)
+				oldchoice = cv_usejoystick2.value;
+
+			if (oldchoice != choice)
+			{
+				if (choice && oldchoice > numjoys) // if we did not select "None", we likely selected a used device
+					CV_SetValue(&cv_usejoystick2, oldchoice);
+
+				if (oldchoice ==
+					(atoi(cv_usejoystick2.string) > numjoys ? atoi(cv_usejoystick2.string) : cv_usejoystick2.value))
+					M_StartMessage("This joystick is used by another\n"
+					               "player. Reset the joystick\n"
+					               "for that player first.\n\n"
+					               "(Press a key)\n", NULL, MM_NOTHING);
+			}
 		}
 	}
 	else
 	{
-		oldchoice = atoi(cv_usejoystick.string) > I_NumJoys() ? atoi(cv_usejoystick.string) : cv_usejoystick.value;
+		oldchoice = atoi(cv_usejoystick.string) > numjoys ? atoi(cv_usejoystick.string) : cv_usejoystick.value;
 		CV_SetValue(&cv_usejoystick, choice);
 
 		// Just in case last-minute changes were made to cv_usejoystick.value,
 		// update the string too
-		CV_SetValue(&cv_usejoystick, cv_usejoystick.value);
-
-		if (oldchoice != choice)
+		// But don't do this if we're intentionally setting higher than numjoys
+		if (choice <= numjoys)
 		{
-			if (choice && oldchoice > I_NumJoys()) // if we did not select "None", we likely selected a used device
-				CV_SetValue(&cv_usejoystick, oldchoice);
+			CV_SetValue(&cv_usejoystick, cv_usejoystick.value);
 
-			if (oldchoice ==
-				(atoi(cv_usejoystick.string) > I_NumJoys() ? atoi(cv_usejoystick.string) : cv_usejoystick.value))
-				M_StartMessage("This joystick is used by another\n"
-				               "player. Reset the joystick\n"
-							   "for that player first.\n\n"
-							   "(Press a key)\n", NULL, MM_NOTHING);
+			// reset this so the comparison is valid
+			if (oldchoice > numjoys)
+				oldchoice = cv_usejoystick.value;
+
+			if (oldchoice != choice)
+			{
+				if (choice && oldchoice > numjoys) // if we did not select "None", we likely selected a used device
+					CV_SetValue(&cv_usejoystick, oldchoice);
+
+				if (oldchoice ==
+					(atoi(cv_usejoystick.string) > numjoys ? atoi(cv_usejoystick.string) : cv_usejoystick.value))
+					M_StartMessage("This joystick is used by another\n"
+					               "player. Reset the joystick\n"
+					               "for that player first.\n\n"
+					               "(Press a key)\n", NULL, MM_NOTHING);
+			}
 		}
 	}
 #else
diff --git a/src/sdl/i_video.c b/src/sdl/i_video.c
index 318777f1..4419284e 100644
--- a/src/sdl/i_video.c
+++ b/src/sdl/i_video.c
@@ -930,12 +930,14 @@ void I_GetEvent(void)
 					// Was cv_usejoystick disabled in settings?
 					if (!strcmp(cv_usejoystick.string, "0") || !cv_usejoystick.value)
 						cv_usejoystick.value = 0;
-					else if (cv_usejoystick.value) // update the cvar ONLY if a device exists
+					else if (atoi(cv_usejoystick.string) <= I_NumJoys() // don't mess if we intentionally set higher than NumJoys
+						     && cv_usejoystick.value) // update the cvar ONLY if a device exists
 						CV_SetValue(&cv_usejoystick, cv_usejoystick.value);
 
 					if (!strcmp(cv_usejoystick2.string, "0") || !cv_usejoystick2.value)
 						cv_usejoystick2.value = 0;
-					else if (cv_usejoystick2.value) // update the cvar ONLY if a device exists
+					else if (atoi(cv_usejoystick2.string) <= I_NumJoys() // don't mess if we intentionally set higher than NumJoys
+					         && cv_usejoystick2.value) // update the cvar ONLY if a device exists
 						CV_SetValue(&cv_usejoystick2, cv_usejoystick2.value);
 
 					// Update all joysticks' init states
@@ -993,12 +995,14 @@ void I_GetEvent(void)
 				// Was cv_usejoystick disabled in settings?
 				if (!strcmp(cv_usejoystick.string, "0"))
 					cv_usejoystick.value = 0;
-				else if (cv_usejoystick.value) // update the cvar ONLY if a device exists
+				else if (atoi(cv_usejoystick.string) <= I_NumJoys() // don't mess if we intentionally set higher than NumJoys
+						 && cv_usejoystick.value) // update the cvar ONLY if a device exists
 					CV_SetValue(&cv_usejoystick, cv_usejoystick.value);
 
 				if (!strcmp(cv_usejoystick2.string, "0"))
 					cv_usejoystick2.value = 0;
-				else if (cv_usejoystick2.value) // update the cvar ONLY if a device exists
+				else if (atoi(cv_usejoystick2.string) <= I_NumJoys() // don't mess if we intentionally set higher than NumJoys
+						 && cv_usejoystick2.value) // update the cvar ONLY if a device exists
 					CV_SetValue(&cv_usejoystick2, cv_usejoystick2.value);
 
 				CONS_Debug(DBG_GAMELOGIC, "Joystick1 device index: %d\n", JoyInfo.oldjoy);