[3.5b1+ Heap Fix] Reset-loop after creating custom arrays

Diamondback

Hello,

me again with array issues... (Sorry ) So after the memory leaks are fixed and the DWC heap access issue is looking good, I'm back with another array related issue.
This time, the board crashes and, more importantly, does not recover anymore. Instead it just appears to sit in a reset loop or something.

I'm currently playing around with creating animations on my 8x8 dotstar "display" and this means I'm creating lots of "image data" in arrays.
Here's an example of some globals that I create:

if !exists(global.displayLEDCount)
	global displayLEDCount = 64
	global displayDefaultBrightness = 10
	global lastDisplayContent = ""
	global toolheadDisplayState = ""
	
if !exists(global.RGBWhite) && boards[0].firmwareVersion != "3.4.5"
	global displayChar0 = {{0,1,1,1,1,1,1,0},{1,1,1,1,1,1,1,1},{1,1,1,0,0,1,1,1},{1,1,1,0,0,1,1,1},{1,1,1,0,0,1,1,1},{1,1,1,0,0,1,1,1},{1,1,1,1,1,1,1,1},{0,1,1,1,1,1,1,0}}
	global displayChar1 = {{0,0,1,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,0,1,1,1,1,1,0},{0,0,1,1,1,0,0,0},{0,0,1,1,1,0,0,0},{0,0,1,1,1,0,0,0},{0,0,1,1,1,0,0,0},{0,0,1,1,1,0,0,0}}	
	global displayChar2 = {{0,1,1,1,1,1,1,0},{1,1,1,1,1,1,1,1},{1,1,1,0,0,1,1,1},{0,1,1,1,0,0,0,0},{0,0,0,1,1,1,0,0},{0,0,0,0,0,1,1,1},{1,1,1,1,1,1,1,1},{1,1,1,1,1,1,1,1}}
	global displayChar3 = {{0,1,1,1,1,1,1,1},{1,1,1,1,1,1,1,1},{1,1,1,0,0,0,0,0},{0,1,1,1,1,1,1,1},{0,1,1,1,1,1,1,1},{1,1,1,0,0,0,0,0},{1,1,1,1,1,1,1,1},{0,1,1,1,1,1,1,1}}
	global displayDigit = {global.displayChar0, global.displayChar1, global.displayChar2, global.displayChar3} 
	
	global FLFrame1 = {{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,1,0,1,1,0,1,0},{1,1,1,1,1,1,1,1},{0,1,0,1,1,0,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global FLFrame2 = {{0,0,0,1,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,1,0,1,1,0,1,0},{1,1,1,1,1,1,1,1},{0,1,0,1,1,0,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global FLFrame3 = {{0,0,0,1,1,0,0,0},{0,0,0,1,0,0,0,0},{0,0,0,0,0,0,0,0},{0,1,0,1,1,0,1,0},{1,1,1,1,1,1,1,1},{0,1,0,1,1,0,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global FLFrame4 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,0,0,0,0},{0,1,0,0,0,0,1,0},{1,1,1,1,1,1,1,1},{0,1,0,1,1,0,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global FLFrame5 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,1,0,1,0,0,1,0},{1,1,1,0,0,1,1,1},{0,1,0,1,1,0,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global FLFrame6 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,1,0,1,1,0,1,0},{1,1,1,1,0,1,1,1},{0,1,0,0,0,0,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global FLFrame7 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,1,0,1,1,0,1,0},{1,1,1,1,1,1,1,1},{0,1,0,1,0,0,1,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0}}
	global FLFrame8 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,1,0,1,1,0,1,0},{1,1,1,1,1,1,1,1},{0,1,0,1,1,0,1,0},{0,0,0,1,0,0,0,0},{0,0,0,0,0,0,0,0}}
	global FLFrame9 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,1,0,1,1,0,1,0},{1,1,1,1,1,1,1,1},{0,1,0,1,1,0,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,0,0,0,0}}
	global filamentLoadFrames = {global.FLFrame1, global.FLFrame2, global.FLFrame3, global.FLFrame4, global.FLFrame5, global.FLFrame6, global.FLFrame7, global.FLFrame8, global.FLFrame9}
	global filamentLoadFrameIndex = 0
		
	global HTFrame1 = {{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global HTFrame2 = {{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0}}
	global HTFrame3 = {{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0}}
	global HTFrame4 = {{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0}}
	global HTFrame5 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0}}
	global HTFrame6 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0}}
	global HTFrame7 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0}}
	global HTFrame8 = {{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global HTFrame9 = {{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global heatingToolFrames = {global.HTFrame1, global.HTFrame2, global.HTFrame3, global.HTFrame4, global.HTFrame5, global.HTFrame6, global.HTFrame7, global.HTFrame8, global.HTFrame9}
	global heatingToolFrameIndex = 0
	
	global HBFrame1 = {{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global HBFrame2 = {{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0}}
	global HBFrame3 = {{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0}}
	global HBFrame4 = {{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0}}
	global HBFrame5 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0}}
	global HBFrame6 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0}}
	global HBFrame7 = {{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0}}
	global HBFrame8 = {{0,0,0,1,1,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global HBFrame9 = {{0,0,0,0,0,0,0,0},{0,0,0,0,0,0,0,0},{0,0,0,1,1,0,0,0},{0,0,1,1,1,1,0,0},{0,1,1,1,1,1,1,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0},{0,0,0,1,1,0,0,0}}
	global heatingBedFrames = {global.HBFrame1, global.HBFrame2, global.HBFrame3, global.HBFrame4, global.HBFrame5, global.HBFrame6, global.HBFrame7, global.HBFrame8, global.HBFrame9}
	global heatingBedFrameIndex = 0
	
	global RGBOff = {0,0,0,0}
	global RGBRed = {255,0,0,25}
	global RGBGreen = {0,255,0,10}
	global RGBBlue = {0,0,255,45}
	global RGBYellow = {255,255,0,30}
	global RGBWhite = {255,255,255,10}	
	global ToolColors = {global.RGBRed, global.RGBGreen, global.RGBYellow, global.RGBBlue}

You can see, lots of wild stuff. Since we currently cannot set individual array element, I have no other way of creating my final "animation" arrays than defining each frame individually and then combining all those frames in another array.
I don't know if that creates another copy of the data or works by reference, but in any case, this will hopefully be replaced by doing it "properly" once we get support for that.

Now, the issue here is that when I execute above code from a macro file, the board appears to crash (DWC disconnects) and then does not come online anymore, DWC keeps trying to connect, but fails to do so in some form of loop. Only disconnecting power brings the board back.

When I remove some of the arrays from the file it works, so to me it feels like something is simply running out of space somewhere. (Duet 3 6HC on @dc42 heap fix build)
Because DWC/the board does not come back online, I can't get a proper M122 after the reset... Would one after power cycling still be useful?

I realize that displaying animations via a 8x8 dotstar grid may not be the intended use case for custom arrays and if there is some sort of hard "don't do that, you stupid", I'm fine with that
However, so far, the board seemed fairly happy with what I was doing, so here we are...

dc42

@Diamondback I assume you are creating all those arrays in code in or called by config.g. Yes a M122 after power cycling would still be useful provided that you haven't done any sort of software reset (M999, Emergency Stop, or firmware upgrade). Alternatively, if you remove the SD card while it is in its reset loop, you will probably be able to connect via USB and do M122 from there.

PS - are you sure the board is resetting? Is the Status LED blinking steadily 0.5sec on and 0.5sec off? If so then it's probably an issue with the arrays being too large to pass back in the object model, and RRF being unable to reply to the HTTP request from DWC.

Diamondback

@dc42 The contents above are all in a single macro file that in this case I execute manually from DWC (for debugging purposes), but yes, ultimately it will be automatic from config.g.

I'll gather a M122 after power cycling then, it's not trivial for me to connect a PC via USB to it, printer is too far away from the desktop, however, if it's gonna be super helpful, I can try to connect my laptop.

Diamondback

@dc42 said in [3.5b1+ Heap Fix] Reset-loop after creating custom arrays:

PS - are you sure the board is resetting? Is the Status LED blinking steadily 0.5sec on and 0.5sec off? If so then it's probably an issue with the arrays being too large to pass back in the object model.

No I'm not at all, I was just assuming by the behavior of DWC. I guess I will do the laptop thing then

dc42

@Diamondback I just tried running that macro on my 6HC bench setup running your config. It doesn't crash for me. However, in the OM viewer only some of the global variables are shown.

Diamondback

@dc42 Try duplicating a bunch of those "frame" arrays, my config.g contains a few other non array globals, maybe there's just some hard limit on the number on globals or soemthing? My code above works for me if I comment line 47 and then it crashes again if I manually send that line via DWC.

Edit: looks like I currently have overall 68 globals in the OM viewer.

dc42

@Diamondback if you open the browser debugging console, you may be able to see why DWC doesn't connect. Possibly because RRF is returning either 501 errors or bad JSON. In my case I think RRF is truncating the info about globals in the OM. It might be that the truncation isn't always clean when the next item due to be added to the reply is a large array.

Diamondback

@dc42 Looks like a 503

PollConnector.ts:251          GET http://192.168.1.31/rr_model?key=global&flags=d99vn 503 (Service Unavailable)
(anonymous) @ PollConnector.ts:251
request @ PollConnector.ts:156
(anonymous) @ PollConnector.ts:211
setTimeout (async)
fe.onload @ PollConnector.ts:210
load (async)
(anonymous) @ PollConnector.ts:157
request @ PollConnector.ts:156
(anonymous) @ PollConnector.ts:203
Promise.then (async)
fe.onload @ PollConnector.ts:201
load (async)
(anonymous) @ PollConnector.ts:157
request @ PollConnector.ts:156
updateLoop @ PollConnector.ts:456
await in updateLoop (async)
doUpdate @ PollConnector.ts:719
setTimeout (async)
scheduleUpdate @ PollConnector.ts:288
reconnect @ PollConnector.ts:372
await in reconnect (async)
reconnect @ index.ts:204
await in reconnect (async)
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ index.ts:222
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
ie.dispatch @ vuex.esm.js:779
onConnectionError @ index.ts:930
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ BaseConnector.ts:160
doUpdate @ PollConnector.ts:721
setTimeout (async)
scheduleUpdate @ PollConnector.ts:288
reconnect @ PollConnector.ts:372
await in reconnect (async)
reconnect @ index.ts:204
await in reconnect (async)
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ index.ts:222
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
ie.dispatch @ vuex.esm.js:779
onConnectionError @ index.ts:930
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ BaseConnector.ts:160
doUpdate @ PollConnector.ts:721
load (async)
(anonymous) @ PollConnector.ts:157
request @ PollConnector.ts:156
(anonymous) @ PollConnector.ts:211
index.ts:916 Error: Operation failed (Reason: Service Unavailable)
    at new ExtendableBuiltin (app.d7c67f8a.js:306:290824)
    at new NetworkError (errors.ts:24:3)
    at new OperationFailedError (errors.ts:50:3)
    at fe.onload (PollConnector.ts:217:14)
onConnectionError @ index.ts:916
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ BaseConnector.ts:160
doUpdate @ PollConnector.ts:721
setTimeout (async)
scheduleUpdate @ PollConnector.ts:288
reconnect @ PollConnector.ts:372
await in reconnect (async)
reconnect @ index.ts:204
await in reconnect (async)
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ index.ts:222
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
ie.dispatch @ vuex.esm.js:779
onConnectionError @ index.ts:930
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ BaseConnector.ts:160
doUpdate @ PollConnector.ts:721
setTimeout (async)
scheduleUpdate @ PollConnector.ts:288
reconnect @ PollConnector.ts:372
await in reconnect (async)
reconnect @ index.ts:204
await in reconnect (async)
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ index.ts:222
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
ie.dispatch @ vuex.esm.js:779
onConnectionError @ index.ts:930
wrappedActionHandler @ vuex.esm.js:851
dispatch @ vuex.esm.js:516
boundDispatch @ vuex.esm.js:406
onConnectionError @ BaseConnector.ts:160
doUpdate @ PollConnector.ts:721
load (async)
(anonymous) @ PollConnector.ts:157
request @ PollConnector.ts:156
(anonymous) @ PollConnector.ts:211

chrishamm

@Diamondback You are using too many and/or too large global variables. Is there any reason you need to make all of them global, wouldn't it suffice to make some of them local using var?

I guess the only way to fix this from our side would be to make model queries of the global key sequential but we need to discuss again how this could be achieved.

Diamondback

@chrishamm Many of my globals are used to store data about filament settings (retraction speed/length, PA tuning etc) and tool settings (nozzle sizes, materials, etc), some of them are multiplied by 4 since I have 4 tools on my TC. Lots of them purely exist because we currently cannot assign individual array elements.

And then there's the "image" data for my 8x8 dotstar grid, where it doesn't really make sense to define the same arrays over and over again, so that's why those are globals.

I can certainly find ways to work around this limitation and once we can set single array elements, the issue will probably go away for me anyway.
Ultimately it still seems like something that should be fixed eventually (or at least changed in some way so that DWC is still usable)

What's the limiting factor here? Overall JSON length of all the OM data? Count of globals?

chrishamm

@Diamondback I was generally referring to your "image" data but of course I don't know where else you reference it. The limiting factor is the maximum output size per (HTTP) request, the JSON response of rr_model?key=global exceeds the current maximum so only HTTP code 503 is returned.

Diamondback

@chrishamm Alright, I will cut some names down in length then and see if I can make it work. As mentioned, at some point this issue should solve itself for me since I then can simply convert a lot of the x4 globals into an array and do some other cleanups.

Thank you