Asynchronous display of two images

[inesmtc]

Hello everyone,

I didn't saw this new forum! Here is a question I posted on the jsPsych google group few days ago :)

For each trial of my experiment, I have to display two half images that form one big image. I've preloaded all my images in the jsPsych.init and all the images' files (left and right) are listed in the timeline variable. My experiment is pretty long (around 300 trials that include the choice screen and a feedback screen) and for some of the trials the feedback images are asynchronously displaid... This is a problem because the participant must see only one big image and not two half images.
This problem happened first using Mozilla. I'm using Chrome and it seemed to work well but I recently had the same issue at the end of my experiment... (for the last 100 trials).
What could be the causes of this? :/ everything is preloaded.. I do not understand. And why is it so random?

Thank you by advance,
Best,
Inès M.

var respS_exp = {
      type: 'html-keyboard-response',
      choices: jsPsych.NO_KEYS,
      trial_duration: feedback_time, 
      timeline: [
      {stimulus: function(){//mask display
          var MasqueL = F2 + SequenceL[SequenceL.length-1].slice(F.length, F.length+3) + '.jpg' ;
          var MasqueR = F2 + SequenceR[SequenceR.length-1].slice(F.length, F.length+3) + '.jpg' ;
          return "<img src=" + MasqueL + " align='center' width=46%  style='z-index:1'></img>"+
          "<img src=" + MasqueR + " align='center' width=46%  style='z-index:1'></img>" 
          },
        trial_duration: mask_time},
      {stimulus: function(){//feedback display
          ImageL = SequenceL[SequenceL.length-1];
          ImageR = SequenceR[SequenceR.length-1];
          if( jsPsych.timelineVariable('rewardL', true)== 1){ 
            hit = 1; 
            let r = Math.random(); 
            if (r<0.8){
              var persoR = ImageR.slice(0,F.length+6);
              var sideR = ImageR.slice(F.length+9, ImageR.length);
              var emoR = 'A_7';
              ImageR = persoR + emoR + sideR;
              rewarded = true;
            }else{
              var persoL = ImageL.slice(0,F.length+6);
              var sideL = ImageL.slice(F.length+9, ImageL.length);
              var emoL = 'A_7';
              ImageL = persoL + emoL + sideL;
              rewarded = false;
            }
          }else{ 
            hit = -1; 
            let r = Math.random(); 
            if (r<0.8){
              var persoL = ImageL.slice(0,F.length+6);
              var sideL = ImageL.slice(F.length+9, ImageL.length);
              var emoL = 'A_7';
              ImageL = persoL + emoL + sideL;
              rewarded = false;
            }else{
              var persoR = ImageR.slice(0,F.length+6);
              var sideR = ImageR.slice(F.length+9, ImageR.length);
              var emoR = 'A_7';
              ImageR = persoR + emoR + sideR;
              rewarded = true;
            }
          }
          return "<img src=" + ImageL + " align='center' width=46%  style='z-index:1'></img>"+
          "<img src='img/CROSS3.png' height='20px' width='20px' style='position:absolute; top: 12.7%; left: 49.2%; z-index:2'></img>"+
          "<img src=" + ImageR + " align='center' width=46%  style='z-index:1'></img>"+
          "<img src='img/check.png' height='80px' width='80px' style='position:absolute; bottom: 45%; left: 13%; z-index:2'></img>"
          }, 
        on_finish: function(data){
          [...]
          }
      }
      ]
    };

[vijaymarupudi]

@jodeleeuw, I can't seem to be able to edit discussions? Anyways, here's the code formatted a bit better.

var respS_exp = {
  type: "html-keyboard-response",
  choices: jsPsych.NO_KEYS,
  trial_duration: feedback_time,
  timeline: [
    {
      stimulus: function() {
        //mask display
        var MasqueL =
          F2 +
          SequenceL[SequenceL.length - 1].slice(F.length, F.length + 3) +
          ".jpg";
        var MasqueR =
          F2 +
          SequenceR[SequenceR.length - 1].slice(F.length, F.length + 3) +
          ".jpg";
        return (
          "<img src=" +
          MasqueL +
          " align='center' width=46%  style='z-index:1'></img>" +
          "<img src=" +
          MasqueR +
          " align='center' width=46%  style='z-index:1'></img>"
        );
      },
      trial_duration: mask_time
    },
    {
      stimulus: function() {
        //feedback display
        ImageL = SequenceL[SequenceL.length - 1];
        ImageR = SequenceR[SequenceR.length - 1];
        if (jsPsych.timelineVariable("rewardL", true) == 1) {
          hit = 1;
          let r = Math.random();
          if (r < 0.8) {
            var persoR = ImageR.slice(0, F.length + 6);
            var sideR = ImageR.slice(F.length + 9, ImageR.length);
            var emoR = "A_7";
            ImageR = persoR + emoR + sideR;
            rewarded = true;
          } else {
            var persoL = ImageL.slice(0, F.length + 6);
            var sideL = ImageL.slice(F.length + 9, ImageL.length);
            var emoL = "A_7";
            ImageL = persoL + emoL + sideL;
            rewarded = false;
          }
        } else {
          hit = -1;
          let r = Math.random();
          if (r < 0.8) {
            var persoL = ImageL.slice(0, F.length + 6);
            var sideL = ImageL.slice(F.length + 9, ImageL.length);
            var emoL = "A_7";
            ImageL = persoL + emoL + sideL;
            rewarded = false;
          } else {
            var persoR = ImageR.slice(0, F.length + 6);
            var sideR = ImageR.slice(F.length + 9, ImageR.length);
            var emoR = "A_7";
            ImageR = persoR + emoR + sideR;
            rewarded = true;
          }
        }
        return (
          "<img src=" +
          ImageL +
          " align='center' width=46%  style='z-index:1'></img>" +
          "<img src='img/CROSS3.png' height='20px' width='20px' style='position:absolute; top: 12.7%; left: 49.2%; z-index:2'></img>" +
          "<img src=" +
          ImageR +
          " align='center' width=46%  style='z-index:1'></img>" +
          "<img src='img/check.png' height='80px' width='80px' style='position:absolute; bottom: 45%; left: 13%; z-index:2'></img>"
        );
      },
      on_finish: function(data) {
        // [...]
      }
    }
  ]
};

[jodeleeuw]

Fixed the permissions issue. You should be able to do this now!

[vijaymarupudi]

Thank you!

[vijaymarupudi]

I don't think there's necessarily a way to fix this without significant code changes. The way jsPsych preloads images means that we cannot access the image cache directly and have to rely on the browser determining that the image is already cached, which causes the lag sometimes. Atleast this is my hypothesis.

jsPsych/jspsych.js

Line 2283 in 8398008

var img_cache = {};

To fix this, you would need to maintain your own image cache and preload mechanism, and refer to it using the on_load callback function. I've linked to the part of jsPsych that might contain hints, but let me know if you'd like more concrete guidance!

[vijaymarupudi]

The variable cache comes from the function

preloadImages(["jsPsych/examples/img/1.gif", "jsPsych/examples/img/2.gif"]).then(function(cache) {
// ----------------->--------------------->----------------->------------------>-------------^
  jsPsych.init({
    timeline: [{
      type: "html-keyboard-response",
      stimulus: '<img src="' + cache["jsPsych/examples/img/1.gif"] + '">'
    }]
  })
})

It is an object with keys being the path to the image and the value being a data url (the cached image).

The image/jpeg part won't result in an error (it just converts it), but feel free to change it to image/png.

[jodeleeuw]

It should be relatively simple to maintain a backwards compatible API while using this technique to provide a user accessible cache! Let me know if you'd like a pull request.

Let's open an issue to track this? It'd be great to try it out and see if the problem is resolved.

[inesmtc]

I received some help from a friend who has more ressources than I do ^^

I put this code at the beginning of mine and put all the jsPsych code (every jsPsych variables declarations and the init) in the 'then' function. I launched it on a local node server (to avoid problems with local images). Some problems occured with the png images but most importantly the asynchronous display remains with firefox...

[vijaymarupudi]

I think that code does the same thing as my code, but is probably more performant.

I think then in this case your images take a lot of time for the browsers to parse. The only solution would be to make your pairs one image. You can either do that outside of code, or inside like this:

function fetchImage(url) {
  const image = new Image();
  return new Promise(resolve => {
    image.onload = () => resolve(image);
    image.src = url;
  });
}

// https://www.tutorialspoint.com/converting-images-to-a-base64-data-url-using-javascript
async function makeSideBySideImage(url1, url2) {
  const [image1, image2] = await Promise.all([
    fetchImage(url1),
    fetchImage(url2)
  ]);

  const canvas = document.createElement("canvas");
  const ctx = canvas.getContext("2d");
  // Set width and height
  canvas.width = image1.width * 2;
  canvas.height = image1.height;
  // Draw the image
  ctx.drawImage(image1, 0, 0);
  ctx.drawImage(image2, image1.width, 0);

  return canvas.toDataURL("image/png");
}

function loadPairs(pairs) {
  return Promise.all(
    pairs.map(pair => makeSideBySideImage(pair.first, pair.second))
  );
}

const pairs = [
  { first: "jsPsych/examples/img/1.gif", second: "jsPsych/examples/img/2.gif" },
  { first: "jsPsych/examples/img/1.gif", second: "jsPsych/examples/img/2.gif" },
  { first: "jsPsych/examples/img/1.gif", second: "jsPsych/examples/img/2.gif" },
  { first: "jsPsych/examples/img/1.gif", second: "jsPsych/examples/img/2.gif" },
  { first: "jsPsych/examples/img/1.gif", second: "jsPsych/examples/img/2.gif" }
];


async function main() {

  const loadedPairs = await loadPairs(pairs);

  jsPsych.init({
    timeline: [
      {
        type: "html-keyboard-response",
        stimulus:
          '<img src="' +
          loadedPairs[0] +
          '">'
      }
    ]
  });
}

main()

[vijaymarupudi]

It should be relatively simple to maintain a backwards compatible API while using this technique to provide a user accessible cache! Let me know if you'd like a pull request.

Let's open an issue to track this? It'd be great to try it out and see if the problem is resolved.

Well in this case it turned out that the problem was not what I initially expected, do you still want to do this?

[inesmtc]

Hi,
I've finally fixed my problem. One of my colleagues noticed that the asynchronous display was due to the fact that one of the two half images was already displayed before the feedback. That is why it was more rapidly displayed. The solution was to display the other half image earlier in the experience hiding it behind the image that we want to show at this moment of the experiment. In this way, we have no more asynchronous display (even with firefox).
Thanks for all your help,
Best,
IM