diff --git a/accessman.go b/accessman.go
index 996d8ad74..aae6d68d4 100644
--- a/accessman.go
+++ b/accessman.go
@@ -545,7 +545,7 @@ func (a *accessMan) checkIncomingConnBanScore(remotePub *btcec.PublicKey) (
 // addPeerAccess tracks a peer's access in the maps. This should be called when
 // the peer has fully connected.
 func (a *accessMan) addPeerAccess(remotePub *btcec.PublicKey,
-	access peerAccessStatus) {
+	access peerAccessStatus, inbound bool) {
 
 	ctx := btclog.WithCtx(
 		context.TODO(), lnutils.LogPubKey("peer", remotePub),
@@ -561,15 +561,37 @@ func (a *accessMan) addPeerAccess(remotePub *btcec.PublicKey,
 
 	a.peerScores[peerMapKey] = peerSlotStatus{state: access}
 
-	// Increment numRestricted.
-	if access == peerStatusRestricted {
+	// Exit early if this is not a restricted peer.
+	if access != peerStatusRestricted {
+		return
+	}
+
+	// Increment numRestricted if this is an inbound connection.
+	if inbound {
 		oldRestricted := a.numRestricted
 		a.numRestricted++
 
 		acsmLog.DebugS(ctx, "Incremented restricted slots",
 			"old_restricted", oldRestricted,
 			"new_restricted", a.numRestricted)
+
+		return
 	}
+
+	// Otherwise, this is a newly created outbound connection. We won't
+	// place any restriction on it, instead, we will do a hot upgrade here
+	// to move it from restricted to temporary.
+	peerCount := channeldb.ChanCount{
+		HasOpenOrClosedChan: false,
+		PendingOpenCount:    0,
+	}
+
+	a.peerCounts[peerMapKey] = peerCount
+	a.peerScores[peerMapKey] = peerSlotStatus{
+		state: peerStatusTemporary,
+	}
+
+	acsmLog.InfoS(ctx, "Upgraded outbound peer: restricted -> temporary")
 }
 
 // removePeerAccess removes the peer's access from the maps. This should be
diff --git a/accessman_test.go b/accessman_test.go
index b67d4f690..30eba194b 100644
--- a/accessman_test.go
+++ b/accessman_test.go
@@ -24,7 +24,7 @@ func assertInboundConnection(t *testing.T, a *accessMan,
 	require.NoError(t, err)
 	require.Equal(t, status, peerAccess)
 
-	a.addPeerAccess(remotePub, peerAccess)
+	a.addPeerAccess(remotePub, peerAccess, true)
 	peerScore, ok := a.peerScores[remotePubSer]
 	require.True(t, ok)
 	require.Equal(t, status, peerScore.state)
diff --git a/server.go b/server.go
index 032d90ae5..639493da6 100644
--- a/server.go
+++ b/server.go
@@ -4316,12 +4316,14 @@ func (s *server) peerConnected(conn net.Conn, connReq *connmgr.ConnReq,
 	addr := conn.RemoteAddr()
 	pubKey := brontideConn.RemotePub()
 
-	// If the remote node's public key is banned, drop the connection.
+	// Only restrict access for inbound connections, which means if the
+	// remote node's public key is banned or the restricted slots are used
+	// up, we will drop the connection.
 	//
 	// TODO(yy): Consider perform this check in
 	// `peerAccessMan.addPeerAccess`.
 	access, err := s.peerAccessMan.assignPeerPerms(pubKey)
-	if err != nil {
+	if inbound && err != nil {
 		pubSer := pubKey.SerializeCompressed()
 
 		// Clean up the persistent peer maps if we're dropping this
@@ -4474,7 +4476,7 @@ func (s *server) peerConnected(conn net.Conn, connReq *connmgr.ConnReq,
 	p := peer.NewBrontide(pCfg)
 
 	// Update the access manager with the access permission for this peer.
-	s.peerAccessMan.addPeerAccess(pubKey, access)
+	s.peerAccessMan.addPeerAccess(pubKey, access, inbound)
 
 	// TODO(roasbeef): update IP address for link-node
 	//  * also mark last-seen, do it one single transaction?