Fixed Mario Kart transparency problem

zsnes/src/video/newgfx16.asm

@@ -425,6 +425,76 @@ BackAreaFillCol dd 0
 clinemainsub    dd 0
 
 BackAreaFill:
+    cmp byte[winbg1enval+eax+5*256],0
+    je near .nowindowb
+    mov ebx,[BackAreaFillCol]
+    cmp ebx,[BackAreaUnFillCol]
+    je near .nowindowb
+    push ecx
+    push edx
+    push eax
+    push edi
+
+    mov ebx,eax
+    mov ecx,[vidbuffer]
+    add ecx,16*2
+    shl eax,9
+    add ecx,eax
+    mov eax,ebx
+    shl eax,6
+    add ecx,eax
+    add ecx,[BackAreaAdd]
+
+    ; Construct Window in ecx
+    mov edi,ngwintable
+    sub ecx,2
+    mov eax,256
+.procnotemptyb
+    mov edx,[edi]
+    add edi,4
+    or edx,edx
+    jz .procemptyb
+    dec edx
+    mov ebx,[BackAreaUnFillCol]
+.swloopb
+    mov dword[ecx],ebx
+    mov dword[ecx+4],ebx
+    add ecx,8
+    sub eax,4
+    jc .doneb
+    sub edx,4
+    jnc .swloopb
+    sub eax,edx
+    add ecx,edx
+    add ecx,edx
+    dec eax
+    add ecx,2
+.procemptyb
+    mov edx,[edi]
+    dec edx
+    add edi,4
+    mov ebx,[BackAreaFillCol]
+.swloop2b
+    mov dword[ecx],ebx
+    mov dword[ecx+4],ebx
+    add ecx,8
+    sub eax,4
+    jc .doneb
+    sub edx,4
+    jnc .swloop2b
+    sub eax,edx
+    add ecx,edx
+    add ecx,edx
+    dec eax
+    add ecx,2
+    jmp .procnotemptyb
+.doneb
+    pop edi
+    pop eax
+    pop edx
+    pop ecx
+    jmp .yeswindowb
+.nowindowb
     push eax
     push ecx
     mov ebx,eax
@@ -437,7 +507,7 @@ BackAreaFill:
     shl eax,6
     add ecx,eax
     add ecx,[BackAreaAdd]
-    mov ebx,[BackAreaFillCol]
+    mov ebx,[BackAreaUnFillCol]
     mov eax,128
 .nowinloop
     mov [ecx],ebx
@@ -1036,24 +1106,49 @@ NEWSYM newengine16b
     mov dword[ngwinen],0
     push ecx
     push edx
-;    cmp byte[winbg1enval+eax+5*256],0
-;    je .nowindowb
-;    mov bl,[winlogicb]
-;    shr bl,2
-;    and bl,03h
-;    mov [nglogicval],bl
-;    mov ebx,5*256
-;    add ebx,eax
-;    call BuildWindow2
+    cmp byte[winbg1enval+eax+5*256],0
+    je .nowindowb
+    mov bl,[winlogicb]
+    shr bl,2
+    and bl,03h
+    mov [nglogicval],bl
+    mov ebx,5*256
+    add ebx,eax
+    call BuildWindow2
 .nowindowb
     mov dword[BackAreaAdd],0
 
     cmp byte[clinemainsub],1
     jne near .domainscreen
     mov ebx,[ColResult]
+    or ebx,[UnusedBit]
+    mov edx,[UnusedBit]
+    mov cl,[scaddset]
 .filledscreen
+    ; get sub-screen colors
+    test cl,10h
+    jnz .inside
+    test cl,20h
+    jnz .outside
+.insideb
+    mov dword[BackAreaUnFillCol],ebx
     mov dword[BackAreaFillCol],ebx
     jmp .donesubscreen
+.inside
+    test cl,20h
+    jnz .filled
+    mov dword[BackAreaUnFillCol],edx
+    mov [BackAreaFillCol],ebx
+    jmp .donesubscreen
+.outside
+    mov [BackAreaUnFillCol],ebx
+    mov dword[BackAreaFillCol],edx
+    jmp .donesubscreen
+.filled
+    xor ebx,ebx
+    mov [BackAreaUnFillCol],edx
+    mov [BackAreaFillCol],edx
+    jmp .donesubscreen
 .domainscreen
     mov ebx,[vbufdptr]
     add ebx,dword[cpalptrng]
@@ -1067,20 +1162,60 @@ NEWSYM newengine16b
     test byte[scaddtype],20h
     jz .notaddbackub
     or ebx,[UnusedBit]
+    mov edx,[UnusedBit]
 .notaddbackub
-    mov dword[BackAreaFillCol],ebx
+    jmp .filledscreen
 .donesubscreen
+    cmp dword[ngwinen],0
+    jne .nowinsc2
+    mov edx,[BackAreaUnFillCol]
+    mov [BackAreaFillCol],edx
+.nowinsc2
+    cmp byte[forceblnk],0
+    je .notforceblanked
+    mov dword[BackAreaUnFillCol],0
+    mov dword[BackAreaFillCol],0
+.notforceblanked
     call BackAreaFill
-
     test byte[FillSubScr+eax],1
     jz near .nosubscreen2
     mov dword[BackAreaAdd],75036*2
     mov ebx,[ColResult]
+    mov edx,[UnusedBit]
     test byte[scaddset],02h
     jz .notbackfixed
     or ebx,[UnusedBit]
 .notbackfixed
+    mov cl,[scaddset]
+    ; get sub-screen colors
+    test cl,10h
+    jnz .inside2
+    test cl,20h
+    jnz .outside2
+.inside2b
+    mov dword[BackAreaUnFillCol],ebx
     mov dword[BackAreaFillCol],ebx
+    jmp .donesubscreen2
+.inside2
+    test cl,20h
+    jnz .filled2
+    mov dword[BackAreaUnFillCol],edx
+    mov [BackAreaFillCol],ebx
+    jmp .donesubscreen2
+.outside2
+    mov [BackAreaUnFillCol],ebx
+    mov dword[BackAreaFillCol],edx
+    jmp .donesubscreen2
+.filled2
+    xor ebx,ebx
+    mov [BackAreaUnFillCol],edx
+    mov [BackAreaFillCol],edx
+.donesubscreen2
+    cmp dword[ngwinen],0
+    jne .nowinsc
+    mov edx,[BackAreaUnFillCol]
+    mov [BackAreaFillCol],edx
+.nowinsc
     call BackAreaFill
 .nosubscreen2
     pop edx
@@ -1220,7 +1355,6 @@ NEWSYM StartDrawNewGfx16b
     mov byte[WindowRedraw],1
     sub dword[endlinet],8
 
-
     ; Clear video memory
 ;    mov edi,[vidbuffer]
 ;    xor eax,eax
@@ -1637,7 +1771,6 @@ NEWSYM StartDrawNewGfx16b
     mov dword[mode0ads],40404040h
     Procbg3pr1b16b 2, drawbg3linepr116b, drawbg3tilepr116b, prdatc, ngmain, 4h
 .nodobg3mb2
-
     call MainScreenClip
     call ProcessTransparencies
 .dontdraw
@@ -2431,7 +2564,6 @@ NEWSYM drawsprngw16b
 .main
     normalwsprng16b sprdrawprawb16bng,sprdrawprbwb16bng
 
-
 drawsprng16bt:
     test byte[scadtng+ebx],10h
     jz near drawsprng16bnt
@@ -2609,6 +2741,7 @@ drawsprngw16bmsnthr:
     xor edi,edi
     normalwsprng16b sprdrawprawb16bngmsnthr,sprdrawprbwb16bngmsnthr
 
+
 ProcessTransparencies:
     cmp byte[NGNoTransp],0
     je .yestransp
@@ -3282,7 +3415,6 @@ NEWSYM HalfTransC, dd 11110111100111101111011110011110b,111101111001111011110111
 NEWSYM NGNoTransp, dd 0
 NEWSYM NewGfx16AsmEnd
 
-
 MainScreenClip:
     mov esi,[vidbuffer]
     add esi,16*2+288*2
@@ -3474,3 +3606,4 @@ SubScreenClip:
     jne near .nextline
     ret
 
+